datachain 0.7.0__tar.gz → 0.7.2__tar.gz

{datachain-0.7.0 → datachain-0.7.2}/.github/workflows/benchmarks.yml

@@ -25,7 +25,7 @@ jobs:
           python-version: '3.12'
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           cache-suffix: benchmarks

{datachain-0.7.0 → datachain-0.7.2}/.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
           python-version: '3.12'
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
 
       - name: Install nox
         run: uv pip install nox --system

{datachain-0.7.0 → datachain-0.7.2}/.github/workflows/tests-studio.yml

@@ -81,7 +81,7 @@ jobs:
           python-version: ${{ matrix.pyv }}
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           cache-suffix: studio

{datachain-0.7.0 → datachain-0.7.2}/.github/workflows/tests.yml

@@ -28,7 +28,7 @@ jobs:
           python-version: '3.9'
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           cache-suffix: lint
@@ -82,7 +82,7 @@ jobs:
           python-version: ${{ matrix.pyv }}
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           cache-suffix: tests-${{ matrix.pyv }}
@@ -142,7 +142,7 @@ jobs:
           python-version: ${{ matrix.pyv }}
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           cache-suffix: examples-${{ matrix.pyv }}

{datachain-0.7.0 → datachain-0.7.2}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.4'
+    rev: 'v0.8.0'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.7.0/src/datachain.egg-info → datachain-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.0
+Version: 0.7.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -98,6 +98,7 @@ Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
 Requires-Dist: pdfplumber==0.11.4; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
+Requires-Dist: ultralytics==8.3.29; extra == "examples"
 
 ================
 |logo| DataChain

datachain-0.7.2/docs/references/sql.md

@@ -0,0 +1,18 @@
+# SQL
+
+Use SQL functions to operate on the underlying database storing the chain data. Useful
+for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter)
+    and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate). Import
+    these functions from `datachain.sql.functions`.
+
+::: datachain.func.avg
+::: datachain.func.count
+::: datachain.func.greatest
+::: datachain.func.least
+::: datachain.func.max
+::: datachain.func.min
+::: datachain.func.rand
+::: datachain.func.sum
+::: datachain.func.array
+::: datachain.func.path
+::: datachain.func.string

{datachain-0.7.0 → datachain-0.7.2}/examples/computer_vision/openimage-detect.py

@@ -1,17 +1,9 @@
 import json
 
 from PIL import Image
-from pydantic import BaseModel
 
-from datachain import C, DataChain, File
-from datachain.sql.functions import path
-
-
-class BBox(BaseModel):
-    x_min: int
-    x_max: int
-    y_min: int
-    y_max: int
+from datachain import C, DataChain, File, model
+from datachain.func import path
 
 
 def openimage_detect(args):
@@ -30,11 +22,13 @@ def openimage_detect(args):
         detections = json.load(stream_json).get("detections", [])
 
     for i, detect in enumerate(detections):
-        bbox = BBox(
-            x_min=int(detect["XMin"] * img.width),
-            x_max=int(detect["XMax"] * img.width),
-            y_min=int(detect["YMin"] * img.height),
-            y_max=int(detect["YMax"] * img.height),
+        bbox = model.BBox.from_list(
+            [
+                detect["XMin"] * img.width,
+                detect["XMax"] * img.width,
+                detect["YMin"] * img.height,
+                detect["YMax"] * img.height,
+            ]
         )
 
         fstream = File(
@@ -54,9 +48,9 @@ source = "gs://datachain-demo/openimages-v6-test-jsonpairs/"
     .filter(C("file.path").glob("*.jpg") | C("file.path").glob("*.json"))
     .agg(
         openimage_detect,
-        partition_by=path.file_stem(C("file.path")),
+        partition_by=path.file_stem("file.path"),
         params=["file"],
-        output={"file": File, "bbox": BBox},
+        output={"file": File, "bbox": model.BBox},
     )
     .show()
 )

datachain-0.7.2/examples/computer_vision/ultralytics-bbox.py

@@ -0,0 +1,22 @@
+from io import BytesIO
+
+from PIL import Image
+from ultralytics import YOLO
+
+from datachain import C, DataChain, File
+from datachain.model.ultralytics import YoloBBoxes
+
+
+def process_bboxes(yolo: YOLO, file: File) -> YoloBBoxes:
+    results = yolo(Image.open(BytesIO(file.read())))
+    return YoloBBoxes.from_results(results)
+
+
+(
+    DataChain.from_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    .filter(C("file.path").glob("*.jpg"))
+    .limit(20)
+    .setup(yolo=lambda: YOLO("yolo11n.pt"))
+    .map(boxes=process_bboxes)
+    .show()
+)

datachain-0.7.2/examples/computer_vision/ultralytics-pose.py

@@ -0,0 +1,22 @@
+from io import BytesIO
+
+from PIL import Image
+from ultralytics import YOLO
+
+from datachain import C, DataChain, File
+from datachain.model.ultralytics import YoloPoses
+
+
+def process_poses(yolo: YOLO, file: File) -> YoloPoses:
+    results = yolo(Image.open(BytesIO(file.read())))
+    return YoloPoses.from_results(results)
+
+
+(
+    DataChain.from_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    .filter(C("file.path").glob("*.jpg"))
+    .limit(20)
+    .setup(yolo=lambda: YOLO("yolo11n-pose.pt"))
+    .map(poses=process_poses)
+    .show()
+)

datachain-0.7.2/examples/computer_vision/ultralytics-segment.py

@@ -0,0 +1,22 @@
+from io import BytesIO
+
+from PIL import Image
+from ultralytics import YOLO
+
+from datachain import C, DataChain, File
+from datachain.model.ultralytics import YoloSegments
+
+
+def process_segments(yolo: YOLO, file: File) -> YoloSegments:
+    results = yolo(Image.open(BytesIO(file.read())))
+    return YoloSegments.from_results(results)
+
+
+(
+    DataChain.from_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    .filter(C("file.path").glob("*.jpg"))
+    .limit(20)
+    .setup(yolo=lambda: YOLO("yolo11n-seg.pt"))
+    .map(segments=process_segments)
+    .show()
+)

{datachain-0.7.0 → datachain-0.7.2}/examples/get_started/common_sql_functions.py

@@ -1,6 +1,5 @@
 from datachain import C, DataChain
-from datachain.sql import literal
-from datachain.sql.functions import array, greatest, least, path, string
+from datachain.func import array, greatest, least, path, string
 
 
 def num_chars_udf(file):
@@ -18,7 +17,7 @@ dc.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).select(
 (
     dc.mutate(
         length=string.length(path.name(C("file.path"))),
-        parts=string.split(path.name(C("file.path")), literal(".")),
+        parts=string.split(path.name(C("file.path")), "."),
     )
     .select("file.path", "length", "parts")
     .show(5)
@@ -35,8 +34,8 @@ dc.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).select(
 
 
 chain = dc.mutate(
-    a=array.length(string.split(C("file.path"), literal("/"))),
-    b=array.length(string.split(path.name(C("file.path")), literal("0"))),
+    a=array.length(string.split("file.path", "/")),
+    b=array.length(string.split(path.name("file.path"), "0")),
 )
 
 (

{datachain-0.7.0 → datachain-0.7.2}/examples/multimodal/clip_inference.py

@@ -3,8 +3,7 @@ import torch
 from torch.nn.functional import cosine_similarity
 from torch.utils.data import DataLoader
 
-from datachain import C, DataChain
-from datachain.sql.functions import path
+from datachain import C, DataChain, func
 
 source = "gs://datachain-demo/50k-laion-files/000000/00000000*"
 
@@ -18,8 +17,8 @@ def create_dataset():
     )
     return imgs.merge(
         captions,
-        on=path.file_stem(imgs.c("file.path")),
-        right_on=path.file_stem(captions.c("file.path")),
+        on=func.path.file_stem(imgs.c("file.path")),
+        right_on=func.path.file_stem(captions.c("file.path")),
     )
 
 

{datachain-0.7.0 → datachain-0.7.2}/examples/multimodal/wds.py

@@ -1,9 +1,9 @@
 import os
 
 from datachain import DataChain
+from datachain.func import path
 from datachain.lib.webdataset import process_webdataset
 from datachain.lib.webdataset_laion import WDSLaion, process_laion_meta
-from datachain.sql.functions import path
 
 IMAGE_TARS = os.getenv(
     "IMAGE_TARS", "gs://datachain-demo/datacomp-small/shards/000000[0-5]*.tar"

{datachain-0.7.0 → datachain-0.7.2}/examples/multimodal/wds_filtered.py

@@ -1,9 +1,7 @@
 import datachain.error
-from datachain import C, DataChain
+from datachain import C, DataChain, func
 from datachain.lib.webdataset import process_webdataset
 from datachain.lib.webdataset_laion import WDSLaion
-from datachain.sql import literal
-from datachain.sql.functions import array, greatest, least, string
 
 name = "wds"
 try:
@@ -20,14 +18,12 @@ except datachain.error.DatasetNotFoundError:
 wds.print_schema()
 
 filtered = (
-    wds.filter(string.length(C("laion.txt")) > 5)
-    .filter(array.length(string.split(C("laion.txt"), literal(" "))) > 2)
+    wds.filter(func.string.length("laion.txt") > 5)
+    .filter(func.array.length(func.string.split("laion.txt", " ")) > 2)
+    .filter(func.least("laion.json.original_width", "laion.json.original_height") > 200)
     .filter(
-        least(C("laion.json.original_width"), C("laion.json.original_height")) > 200
-    )
-    .filter(
-        greatest(C("laion.json.original_width"), C("laion.json.original_height"))
-        / least(C("laion.json.original_width"), C("laion.json.original_height"))
+        func.greatest("laion.json.original_width", "laion.json.original_height")
+        / func.least("laion.json.original_width", "laion.json.original_height")
         < 3.0
     )
     .save()

{datachain-0.7.0 → datachain-0.7.2}/pyproject.toml

@@ -110,7 +110,8 @@ examples = [
   "unstructured[pdf,embed-huggingface]<0.16.0",
   "pdfplumber==0.11.4",
   "huggingface_hub[hf_transfer]",
-  "onnx==1.16.1"
+  "onnx==1.16.1",
+  "ultralytics==8.3.29"
 ]
 
 [project.urls]

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/__init__.py

@@ -1,4 +1,3 @@
-from datachain.lib import func, models
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import C, Column, DataChain, Sys
 from datachain.lib.file import (
@@ -35,9 +34,7 @@ __all__ = [
     "Sys",
     "TarVFile",
     "TextFile",
-    "func",
     "is_chain_type",
     "metrics",
-    "models",
     "param",
 ]

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/catalog/catalog.py

@@ -54,7 +54,6 @@ from datachain.error import (
     QueryScriptCancelError,
     QueryScriptRunError,
 )
-from datachain.listing import Listing
 from datachain.node import DirType, Node, NodeWithPath
 from datachain.nodes_thread_pool import NodesThreadPool
 from datachain.remote.studio import StudioClient
@@ -76,6 +75,7 @@ if TYPE_CHECKING:
     from datachain.dataset import DatasetVersion
     from datachain.job import Job
     from datachain.lib.file import File
+    from datachain.listing import Listing
 
 logger = logging.getLogger("datachain")
 
@@ -236,7 +236,7 @@ class DatasetRowsFetcher(NodesThreadPool):
 class NodeGroup:
     """Class for a group of nodes from the same source"""
 
-    listing: Listing
+    listing: "Listing"
     sources: list[DataSource]
 
     # The source path within the bucket
@@ -591,8 +591,9 @@ class Catalog:
         client_config=None,
         object_name="file",
         skip_indexing=False,
-    ) -> tuple[Listing, str]:
+    ) -> tuple["Listing", str]:
         from datachain.lib.dc import DataChain
+        from datachain.listing import Listing
 
         DataChain.from_storage(
             source, session=self.session, update=update, object_name=object_name
@@ -660,7 +661,8 @@ class Catalog:
         no_glob: bool = False,
         client_config=None,
     ) -> list[NodeGroup]:
-        from datachain.query import DatasetQuery
+        from datachain.listing import Listing
+        from datachain.query.dataset import DatasetQuery
 
         def _row_to_node(d: dict[str, Any]) -> Node:
             del d["file__source"]
@@ -876,7 +878,7 @@ class Catalog:
     def update_dataset_version_with_warehouse_info(
         self, dataset: DatasetRecord, version: int, rows_dropped=False, **kwargs
     ) -> None:
-        from datachain.query import DatasetQuery
+        from datachain.query.dataset import DatasetQuery
 
         dataset_version = dataset.get_version(version)
 
@@ -1177,7 +1179,7 @@ class Catalog:
     def ls_dataset_rows(
         self, name: str, version: int, offset=None, limit=None
     ) -> list[dict]:
-        from datachain.query import DatasetQuery
+        from datachain.query.dataset import DatasetQuery
 
         dataset = self.get_dataset(name)
 

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/cli.py

@@ -957,7 +957,7 @@ def show(
     schema: bool = False,
 ) -> None:
     from datachain.lib.dc import DataChain
-    from datachain.query import DatasetQuery
+    from datachain.query.dataset import DatasetQuery
     from datachain.utils import show_records
 
     dataset = catalog.get_dataset(name)

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/client/fsspec.py

@@ -28,7 +28,6 @@ from tqdm import tqdm
 from datachain.cache import DataChainCache
 from datachain.client.fileslice import FileWrapper
 from datachain.error import ClientError as DataChainClientError
-from datachain.lib.file import File
 from datachain.nodes_fetcher import NodesFetcher
 from datachain.nodes_thread_pool import NodeChunk
 
@@ -36,6 +35,7 @@ if TYPE_CHECKING:
     from fsspec.spec import AbstractFileSystem
 
     from datachain.dataset import StorageURI
+    from datachain.lib.file import File
 
 
 logger = logging.getLogger("datachain")
@@ -45,7 +45,7 @@ DELIMITER = "/"  # Path delimiter.
 
 DATA_SOURCE_URI_PATTERN = re.compile(r"^[\w]+:\/\/.*$")
 
-ResultQueue = asyncio.Queue[Optional[Sequence[File]]]
+ResultQueue = asyncio.Queue[Optional[Sequence["File"]]]
 
 
 def _is_win_local_path(uri: str) -> bool:
@@ -212,7 +212,7 @@ class Client(ABC):
 
     async def scandir(
         self, start_prefix: str, method: str = "default"
-    ) -> AsyncIterator[Sequence[File]]:
+    ) -> AsyncIterator[Sequence["File"]]:
         try:
             impl = getattr(self, f"_fetch_{method}")
         except AttributeError:
@@ -317,7 +317,7 @@ class Client(ABC):
         return f"{self.PREFIX}{self.name}/{rel_path}"
 
     @abstractmethod
-    def info_to_file(self, v: dict[str, Any], parent: str) -> File: ...
+    def info_to_file(self, v: dict[str, Any], parent: str) -> "File": ...
 
     def fetch_nodes(
         self,
@@ -354,7 +354,7 @@ class Client(ABC):
             copy2(src, dst)
 
     def open_object(
-        self, file: File, use_cache: bool = True, cb: Callback = DEFAULT_CALLBACK
+        self, file: "File", use_cache: bool = True, cb: Callback = DEFAULT_CALLBACK
     ) -> BinaryIO:
         """Open a file, including files in tar archives."""
         if use_cache and (cache_path := self.cache.get_path(file)):
@@ -362,19 +362,19 @@ class Client(ABC):
         assert not file.location
         return FileWrapper(self.fs.open(self.get_full_path(file.path)), cb)  # type: ignore[return-value]
 
-    def download(self, file: File, *, callback: Callback = DEFAULT_CALLBACK) -> None:
+    def download(self, file: "File", *, callback: Callback = DEFAULT_CALLBACK) -> None:
         sync(get_loop(), functools.partial(self._download, file, callback=callback))
 
-    async def _download(self, file: File, *, callback: "Callback" = None) -> None:
+    async def _download(self, file: "File", *, callback: "Callback" = None) -> None:
         if self.cache.contains(file):
             # Already in cache, so there's nothing to do.
             return
         await self._put_in_cache(file, callback=callback)
 
-    def put_in_cache(self, file: File, *, callback: "Callback" = None) -> None:
+    def put_in_cache(self, file: "File", *, callback: "Callback" = None) -> None:
         sync(get_loop(), functools.partial(self._put_in_cache, file, callback=callback))
 
-    async def _put_in_cache(self, file: File, *, callback: "Callback" = None) -> None:
+    async def _put_in_cache(self, file: "File", *, callback: "Callback" = None) -> None:
         assert not file.location
         if file.etag:
             etag = await self.get_current_etag(file)

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/data_storage/schema.py

@@ -12,7 +12,7 @@ import sqlalchemy as sa
 from sqlalchemy.sql import func as f
 from sqlalchemy.sql.expression import false, null, true
 
-from datachain.sql.functions import path
+from datachain.sql.functions import path as pathfunc
 from datachain.sql.types import Int, SQLType, UInt64
 
 if TYPE_CHECKING:
@@ -130,7 +130,7 @@ class DirExpansion:
 
     def query(self, q):
         q = self.base_select(q).cte(recursive=True)
-        parent = path.parent(self.c(q, "path"))
+        parent = pathfunc.parent(self.c(q, "path"))
         q = q.union_all(
             sa.select(
                 sa.literal(-1).label("sys__id"),

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/data_storage/sqlite.py

@@ -122,7 +122,9 @@ class SQLiteDatabaseEngine(DatabaseEngine):
         return cls(*cls._connect(db_file=db_file))
 
     @staticmethod
-    def _connect(db_file: Optional[str] = None):
+    def _connect(
+        db_file: Optional[str] = None,
+    ) -> tuple["Engine", "MetaData", sqlite3.Connection, str]:
         try:
             if db_file == ":memory:":
                 # Enable multithreaded usage of the same in-memory db
@@ -130,9 +132,8 @@ class SQLiteDatabaseEngine(DatabaseEngine):
                     _get_in_memory_uri(), uri=True, detect_types=DETECT_TYPES
                 )
             else:
-                db = sqlite3.connect(
-                    db_file or DataChainDir.find().db, detect_types=DETECT_TYPES
-                )
+                db_file = db_file or DataChainDir.find().db
+                db = sqlite3.connect(db_file, detect_types=DETECT_TYPES)
             create_user_defined_sql_functions(db)
             engine = sqlalchemy.create_engine(
                 "sqlite+pysqlite:///", creator=lambda: db, future=True

{datachain-0.7.0 → datachain-0.7.2}/src/datachain/data_storage/warehouse.py

@@ -224,28 +224,28 @@ class AbstractWarehouse(ABC, Serializable):
         offset = 0
         num_yielded = 0
 
-        while True:
-            if limit is not None:
-                limit -= num_yielded
-                if limit == 0:
-                    break
-                if limit < page_size:
-                    paginated_query = paginated_query.limit(None).limit(limit)
-
-            # Ensure we're using a thread-local connection
-            with self.clone() as wh:
+        # Ensure we're using a thread-local connection
+        with self.clone() as wh:
+            while True:
+                if limit is not None:
+                    limit -= num_yielded
+                    if limit == 0:
+                        break
+                    if limit < page_size:
+                        paginated_query = paginated_query.limit(None).limit(limit)
+
                 # Cursor results are not thread-safe, so we convert them to a list
                 results = list(wh.dataset_rows_select(paginated_query.offset(offset)))
 
-            processed = False
-            for row in results:
-                processed = True
-                yield row
-                num_yielded += 1
+                processed = False
+                for row in results:
+                    processed = True
+                    yield row
+                    num_yielded += 1
 
-            if not processed:
-                break  # no more results
-            offset += page_size
+                if not processed:
+                    break  # no more results
+                offset += page_size
 
     #
     # Table Name Internal Functions

datachain-0.7.2/src/datachain/func/__init__.py

@@ -0,0 +1,49 @@
+from sqlalchemy import literal
+
+from . import array, path, random, string
+from .aggregate import (
+    any_value,
+    avg,
+    collect,
+    concat,
+    count,
+    dense_rank,
+    first,
+    max,
+    min,
+    rank,
+    row_number,
+    sum,
+)
+from .array import cosine_distance, euclidean_distance, length, sip_hash_64
+from .conditional import greatest, least
+from .random import rand
+from .window import window
+
+__all__ = [
+    "any_value",
+    "array",
+    "avg",
+    "collect",
+    "concat",
+    "cosine_distance",
+    "count",
+    "dense_rank",
+    "euclidean_distance",
+    "first",
+    "greatest",
+    "least",
+    "length",
+    "literal",
+    "max",
+    "min",
+    "path",
+    "rand",
+    "random",
+    "rank",
+    "row_number",
+    "sip_hash_64",
+    "string",
+    "sum",
+    "window",
+]