datachain 0.3.18__tar.gz → 0.3.20__tar.gz

{datachain-0.3.18 → datachain-0.3.20}/.github/workflows/benchmarks.yml

@@ -23,15 +23,18 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: benchmarks
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox and dvc
+        run: uv pip install dvc[gs] nox --system
 
-      - run: uv pip install dvc[gs] --system
-      - run: dvc --cd tests/benchmarks/datasets pull
+      - name: Pull dataset
+        run: dvc --cd tests/benchmarks/datasets pull
       - name: Run benchmarks
         run: nox -s bench

{datachain-0.3.18 → datachain-0.3.20}/.github/workflows/release.yml

@@ -21,17 +21,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
 
+      - name: Install nox
+        run: uv pip install nox --system
       - name: Build package
         run: nox -s build
 

{datachain-0.3.18 → datachain-0.3.20}/.github/workflows/tests-studio.yml

@@ -82,12 +82,15 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Install uv
-        run: |
-          python -m pip install --upgrade uv
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: studio
+          cache-dependency-glob: |
+            backend/datachain_server/pyproject.toml
+            backend/datachain/pyproject.toml
 
       - name: Install dependencies
         run: uv pip install --system ./backend/datachain_server[tests] ./backend/datachain[tests]

{datachain-0.3.18 → datachain-0.3.20}/.github/workflows/tests.yml

@@ -26,13 +26,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.9'
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: lint
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Cache mypy
         uses: actions/cache@v4
@@ -77,13 +80,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: tests-${{ matrix.pyv }}
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Skip flaky azure, gs remotes on macOS
         if: runner.os == 'macOS'
@@ -134,13 +140,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: examples-${{ matrix.pyv }}
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Run examples
         run: nox -s examples -p ${{ matrix.pyv }} -- -m "${{ matrix.group }}"

{datachain-0.3.18/src/datachain.egg-info → datachain-0.3.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.18
+Version: 0.3.20
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -34,7 +34,6 @@ Requires-Dist: dvc-objects<6,>=4
 Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
-Requires-Dist: dill==0.3.8
 Requires-Dist: cloudpickle
 Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic<3,>=2

{datachain-0.3.18 → datachain-0.3.20}/docs/references/file.md

@@ -7,6 +7,8 @@ automatically when creating a `DataChain` from files, like in
 classes include various metadata fields about the underlying file as well as methods to
 read from the files and otherwise work with the file contents.
 
+::: datachain.lib.file.ArrowRow
+
 ::: datachain.lib.file.ExportPlacement
 
 ::: datachain.lib.file.File
@@ -15,8 +17,6 @@ read from the files and otherwise work with the file contents.
 
 ::: datachain.lib.file.ImageFile
 
-::: datachain.lib.file.IndexedFile
-
 ::: datachain.lib.file.TarVFile
 
 ::: datachain.lib.file.TextFile

{datachain-0.3.18 → datachain-0.3.20}/noxfile.py

@@ -57,8 +57,8 @@ def lint(session: nox.Session) -> None:
 
 @nox.session
 def build(session: nox.Session) -> None:
-    session.install("build", "twine", "uv")
-    session.run("python", "-m", "build", "--installer", "uv")
+    session.install("twine", "uv")
+    session.run("uv", "build")
     dists = glob.glob("dist/*")
     session.run("twine", "check", *dists, silent=True)
 

{datachain-0.3.18 → datachain-0.3.20}/pyproject.toml

@@ -37,7 +37,6 @@ dependencies = [
   "shtab>=1.3.4,<2",
   "sqlalchemy>=2",
   "multiprocess==0.70.16",
-  "dill==0.3.8",
   "cloudpickle",
   "orjson>=3.10.5",
   "pydantic>=2,<3",

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/__init__.py

@@ -1,21 +1,23 @@
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import C, Column, DataChain, Sys
 from datachain.lib.file import (
+    ArrowRow,
     File,
     FileError,
     ImageFile,
-    IndexedFile,
     TarVFile,
     TextFile,
 )
 from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
+from datachain.query import metrics, param
 from datachain.query.session import Session
 
 __all__ = [
     "AbstractUDF",
     "Aggregator",
+    "ArrowRow",
     "C",
     "Column",
     "DataChain",
@@ -26,7 +28,6 @@ __all__ = [
     "FileError",
     "Generator",
     "ImageFile",
-    "IndexedFile",
     "Mapper",
     "ModelStore",
     "Session",
@@ -34,4 +35,6 @@ __all__ = [
     "TarVFile",
     "TextFile",
     "is_chain_type",
+    "metrics",
+    "param",
 ]

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/catalog/catalog.py

@@ -1,4 +1,3 @@
-import ast
 import glob
 import io
 import json
@@ -53,9 +52,9 @@ from datachain.error import (
     DataChainError,
     DatasetInvalidVersionError,
     DatasetNotFoundError,
+    DatasetVersionNotFoundError,
     PendingIndexingError,
     QueryScriptCancelError,
-    QueryScriptCompileError,
     QueryScriptRunError,
 )
 from datachain.listing import Listing
@@ -80,6 +79,7 @@ if TYPE_CHECKING:
     )
     from datachain.dataset import DatasetVersion
     from datachain.job import Job
+    from datachain.lib.file import File
 
 logger = logging.getLogger("datachain")
 
@@ -588,37 +588,6 @@ class Catalog:
     def generate_query_dataset_name(cls) -> str:
         return f"{QUERY_DATASET_PREFIX}_{uuid4().hex}"
 
-    def attach_query_wrapper(self, code_ast):
-        if code_ast.body:
-            last_expr = code_ast.body[-1]
-            if isinstance(last_expr, ast.Expr):
-                new_expressions = [
-                    ast.Import(
-                        names=[ast.alias(name="datachain.query.dataset", asname=None)]
-                    ),
-                    ast.Expr(
-                        value=ast.Call(
-                            func=ast.Attribute(
-                                value=ast.Attribute(
-                                    value=ast.Attribute(
-                                        value=ast.Name(id="datachain", ctx=ast.Load()),
-                                        attr="query",
-                                        ctx=ast.Load(),
-                                    ),
-                                    attr="dataset",
-                                    ctx=ast.Load(),
-                                ),
-                                attr="query_wrapper",
-                                ctx=ast.Load(),
-                            ),
-                            args=[last_expr],
-                            keywords=[],
-                        )
-                    ),
-                ]
-                code_ast.body[-1:] = new_expressions
-        return code_ast
-
     def get_client(self, uri: str, **config: Any) -> Client:
         """
         Return the client corresponding to the given source `uri`.
@@ -1218,7 +1187,9 @@ class Catalog:
 
         dataset_version = dataset.get_version(version)
         if not dataset_version:
-            raise ValueError(f"Dataset {dataset.name} does not have version {version}")
+            raise DatasetVersionNotFoundError(
+                f"Dataset {dataset.name} does not have version {version}"
+            )
 
         if not dataset_version.is_final_status():
             raise ValueError("Cannot register dataset version in non final status")
@@ -1429,65 +1400,34 @@ class Catalog:
         dataset = self.get_dataset(name)
         return self.update_dataset(dataset, **update_data)
 
-    def get_file_signals(
-        self, dataset_name: str, dataset_version: int, row: RowDict
-    ) -> Optional[RowDict]:
+    def get_file_from_row(
+        self, dataset_name: str, dataset_version: int, row: RowDict, signal_name: str
+    ) -> "File":
         """
-        Function that returns file signals from dataset row.
-        Note that signal names are without prefix, so if there was 'laion__file__source'
-        in original row, result will have just 'source'
-        Example output:
-            {
-                "source": "s3://ldb-public",
-                "path": "animals/dogs/dog.jpg",
-                ...
-            }
+        Function that returns specific file signal from dataset row by name.
         """
         from datachain.lib.file import File
         from datachain.lib.signal_schema import DEFAULT_DELIMITER, SignalSchema
 
         version = self.get_dataset(dataset_name).get_version(dataset_version)
-
-        file_signals_values = RowDict()
-
         schema = SignalSchema.deserialize(version.feature_schema)
-        for file_signals in schema.get_signals(File):
-            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
-            file_signals_values[file_signals] = {
-                c_name.removeprefix(prefix): c_value
-                for c_name, c_value in row.items()
-                if c_name.startswith(prefix)
-                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
-            }
-
-        if not file_signals_values:
-            return None
 
-        # there can be multiple file signals in a schema, but taking the first
-        # one for now. In future we might add ability to choose from which one
-        # to open object
-        return next(iter(file_signals_values.values()))
-
-    def open_object(
-        self,
-        dataset_name: str,
-        dataset_version: int,
-        row: RowDict,
-        use_cache: bool = True,
-        **config: Any,
-    ):
-        from datachain.lib.file import File
+        if signal_name not in schema.get_signals(File):
+            raise RuntimeError(
+                f"File signal with path {signal_name} not found in ",
+                f"dataset {dataset_name}@v{dataset_version} signals schema",
+            )
 
-        file_signals = self.get_file_signals(dataset_name, dataset_version, row)
-        if not file_signals:
-            raise RuntimeError("Cannot open object without file signals")
+        prefix = signal_name.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
+        file_signals = {
+            c_name.removeprefix(prefix): c_value
+            for c_name, c_value in row.items()
+            if c_name.startswith(prefix)
+            and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
+            and c_name.removeprefix(prefix) in File.model_fields
+        }
 
-        config = config or self.client_config
-        client = self.get_client(file_signals["source"], **config)
-        return client.open_object(
-            File._from_row(file_signals),
-            use_cache=use_cache,
-        )
+        return File(**file_signals)
 
     def ls(
         self,
@@ -1581,7 +1521,7 @@ class Catalog:
 
         try:
             remote_dataset_version = remote_dataset.get_version(version)
-        except (ValueError, StopIteration) as exc:
+        except (DatasetVersionNotFoundError, StopIteration) as exc:
             raise DataChainError(
                 f"Dataset {remote_dataset_name} doesn't have version {version}"
                 " on server"
@@ -1722,64 +1662,24 @@ class Catalog:
         query_script: str,
         env: Optional[Mapping[str, str]] = None,
         python_executable: str = sys.executable,
-        save: bool = False,
-        capture_output: bool = True,
+        capture_output: bool = False,
         output_hook: Callable[[str], None] = noop,
         params: Optional[dict[str, str]] = None,
         job_id: Optional[str] = None,
-        _execute_last_expression: bool = False,
     ) -> None:
-        """
-        Method to run custom user Python script to run a query and, as result,
-        creates new dataset from the results of a query.
-        Returns tuple of result dataset and script output.
-
-        Constraints on query script:
-            1. datachain.query.DatasetQuery should be used in order to create query
-            for a dataset
-            2. There should not be any .save() call on DatasetQuery since the idea
-            is to create only one dataset as the outcome of the script
-            3. Last statement must be an instance of DatasetQuery
-
-        If save is set to True, we are creating new dataset with results
-        from dataset query. If it's set to False, we will just print results
-        without saving anything
-
-        Example of query script:
-            from datachain.query import DatasetQuery, C
-            DatasetQuery('s3://ldb-public/remote/datasets/mnist-tiny/').filter(
-                C.size > 1000
-            )
-        """
-        if _execute_last_expression:
-            try:
-                code_ast = ast.parse(query_script)
-                code_ast = self.attach_query_wrapper(code_ast)
-                query_script_compiled = ast.unparse(code_ast)
-            except Exception as exc:
-                raise QueryScriptCompileError(
-                    f"Query script failed to compile, reason: {exc}"
-                ) from exc
-        else:
-            query_script_compiled = query_script
-            assert not save
-
+        cmd = [python_executable, "-c", query_script]
         env = dict(env or os.environ)
         env.update(
             {
                 "DATACHAIN_QUERY_PARAMS": json.dumps(params or {}),
-                "PYTHONPATH": os.getcwd(),  # For local imports
-                "DATACHAIN_QUERY_SAVE": "1" if save else "",
-                "PYTHONUNBUFFERED": "1",
                 "DATACHAIN_JOB_ID": job_id or "",
             },
         )
-        popen_kwargs = {}
+        popen_kwargs: dict[str, Any] = {}
         if capture_output:
             popen_kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT}
 
-        cmd = [python_executable, "-c", query_script_compiled]
-        with subprocess.Popen(cmd, env=env, **popen_kwargs) as proc:  # type: ignore[call-overload]  # noqa: S603
+        with subprocess.Popen(cmd, env=env, **popen_kwargs) as proc:  # noqa: S603
             if capture_output:
                 args = (proc.stdout, output_hook)
                 thread = Thread(target=_process_stream, args=args, daemon=True)

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/cli.py

@@ -804,7 +804,6 @@ def query(
         catalog.query(
             script_content,
             python_executable=python_executable,
-            capture_output=False,
             params=params,
             job_id=job_id,
         )

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/dataset.py

@@ -12,6 +12,7 @@ from typing import (
 from urllib.parse import urlparse
 
 from datachain.client import Client
+from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
 if TYPE_CHECKING:
@@ -417,7 +418,9 @@ class DatasetRecord:
 
     def get_version(self, version: int) -> DatasetVersion:
         if not self.has_version(version):
-            raise ValueError(f"Dataset {self.name} does not have version {version}")
+            raise DatasetVersionNotFoundError(
+                f"Dataset {self.name} does not have version {version}"
+            )
         return next(
             v
             for v in self.versions  # type: ignore [union-attr]
@@ -435,7 +438,9 @@ class DatasetRecord:
         Get identifier in the form my-dataset@v3
         """
         if not self.has_version(version):
-            raise ValueError(f"Dataset {self.name} doesn't have a version {version}")
+            raise DatasetVersionNotFoundError(
+                f"Dataset {self.name} doesn't have a version {version}"
+            )
         return f"{self.name}@v{version}"
 
     def uri(self, version: int) -> str:

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/error.py

@@ -10,6 +10,10 @@ class DatasetNotFoundError(NotFoundError):
     pass
 
 
+class DatasetVersionNotFoundError(NotFoundError):
+    pass
+
+
 class DatasetInvalidVersionError(Exception):
     pass
 
@@ -32,14 +36,12 @@ class QueryScriptRunError(Exception):
     Attributes:
         message      Explanation of the error
         return_code  Code returned by the subprocess
-        output       STDOUT + STDERR output of the subprocess
     """
 
-    def __init__(self, message: str, return_code: int = 0, output: str = ""):
+    def __init__(self, message: str, return_code: int = 0):
         self.message = message
         self.return_code = return_code
-        self.output = output
-        super().__init__(self.message)
+        super().__init__(message)
 
 
 class QueryScriptCancelError(QueryScriptRunError):

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/lib/arrow.py

@@ -4,11 +4,11 @@ from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Optional
 
 import pyarrow as pa
-from pyarrow.dataset import dataset
+from pyarrow.dataset import CsvFileFormat, dataset
 from tqdm import tqdm
 
 from datachain.lib.data_model import dict_to_data_model
-from datachain.lib.file import File, IndexedFile
+from datachain.lib.file import ArrowRow, File
 from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import Generator
 
@@ -84,7 +84,12 @@ class ArrowGenerator(Generator):
                                 vals_dict[field] = val
                         vals = [self.output_schema(**vals_dict)]
                     if self.source:
-                        yield [IndexedFile(file=file, index=index), *vals]
+                        kwargs: dict = self.kwargs
+                        # Can't serialize CsvFileFormat; may lose formatting options.
+                        if isinstance(kwargs.get("format"), CsvFileFormat):
+                            kwargs["format"] = "csv"
+                        arrow_file = ArrowRow(file=file, index=index, kwargs=kwargs)
+                        yield [arrow_file, *vals]
                     else:
                         yield vals
                     index += 1

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/lib/dc.py

@@ -26,8 +26,8 @@ from datachain.lib.convert.python_to_sql import python_to_sql
 from datachain.lib.convert.values_to_tuples import values_to_tuples
 from datachain.lib.data_model import DataModel, DataType, dict_to_data_model
 from datachain.lib.dataset_info import DatasetInfo
+from datachain.lib.file import ArrowRow, File, get_file_type
 from datachain.lib.file import ExportPlacement as FileExportPlacement
-from datachain.lib.file import File, IndexedFile, get_file_type
 from datachain.lib.listing import (
     is_listing_dataset,
     is_listing_expired,
@@ -1614,7 +1614,7 @@ class DataChain(DatasetQuery):
                 for name, info in output.model_fields.items()
             }
         if source:
-            output = {"source": IndexedFile} | output  # type: ignore[assignment,operator]
+            output = {"source": ArrowRow} | output  # type: ignore[assignment,operator]
         return self.gen(
             ArrowGenerator(schema, model, source, nrows, **kwargs), output=output
         )

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/lib/file.py

@@ -17,6 +17,7 @@ from urllib.request import url2pathname
 
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
 from PIL import Image
+from pyarrow.dataset import dataset
 from pydantic import Field, field_validator
 
 if TYPE_CHECKING:
@@ -439,14 +440,31 @@ class ImageFile(File):
         self.read().save(destination)
 
 
-class IndexedFile(DataModel):
-    """Metadata indexed from tabular files.
-
-    Includes `file` and `index` signals.
-    """
+class ArrowRow(DataModel):
+    """`DataModel` for reading row from Arrow-supported file."""
 
     file: File
     index: int
+    kwargs: dict
+
+    @contextmanager
+    def open(self):
+        """Stream row contents from indexed file."""
+        if self.file._caching_enabled:
+            self.file.ensure_cached()
+            path = self.file.get_local_path()
+            ds = dataset(path, **self.kwargs)
+
+        else:
+            path = self.file.get_path()
+            ds = dataset(path, filesystem=self.file.get_fs(), **self.kwargs)
+
+        return ds.take([self.index]).to_reader()
+
+    def read(self):
+        """Returns row contents as dict."""
+        with self.open() as record_batch:
+            return record_batch.to_pylist()[0]
 
 
 def get_file_type(type_: Literal["binary", "text", "image"] = "binary") -> type[File]:

{datachain-0.3.18 → datachain-0.3.20}/src/datachain/query/dataset.py

@@ -1604,25 +1604,3 @@ class DatasetQuery:
         finally:
             self.cleanup()
         return self.__class__(name=name, version=version, catalog=self.catalog)
-
-
-def query_wrapper(dataset_query: Any) -> Any:
-    """
-    Wrapper function that wraps the last statement of user query script.
-    Last statement MUST be instance of DatasetQuery, otherwise script exits with
-    error code 10
-    """
-    if not isinstance(dataset_query, DatasetQuery):
-        return dataset_query
-
-    catalog = dataset_query.catalog
-    save = bool(os.getenv("DATACHAIN_QUERY_SAVE"))
-
-    is_session_temp_dataset = dataset_query.name and dataset_query.name.startswith(
-        dataset_query.session.get_temp_prefix()
-    )
-
-    if save and (is_session_temp_dataset or not dataset_query.attached):
-        name = catalog.generate_query_dataset_name()
-        dataset_query = dataset_query.save(name)
-    return dataset_query