datachain 0.25.1__tar.gz → 0.26.0__tar.gz

{datachain-0.25.1 → datachain-0.26.0}/.gitignore

@@ -143,3 +143,5 @@ cython_debug/
 
 # pt files produced by ultralytics examples
 *.pt
+
+.DS_Store/

{datachain-0.25.1 → datachain-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.25.1
+Version: 0.26.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -63,6 +63,9 @@ Provides-Extra: torch
 Requires-Dist: torch>=2.1.0; extra == "torch"
 Requires-Dist: torchvision; extra == "torch"
 Requires-Dist: transformers>=4.36.0; extra == "torch"
+Provides-Extra: audio
+Requires-Dist: torchaudio; extra == "audio"
+Requires-Dist: soundfile; extra == "audio"
 Provides-Extra: remote
 Requires-Dist: lz4; extra == "remote"
 Requires-Dist: requests>=2.22.0; extra == "remote"
@@ -78,7 +81,7 @@ Requires-Dist: ffmpeg-python; extra == "video"
 Requires-Dist: imageio[ffmpeg,pyav]>=2.37.0; extra == "video"
 Requires-Dist: opencv-python; extra == "video"
 Provides-Extra: tests
-Requires-Dist: datachain[hf,remote,torch,vector,video]; extra == "tests"
+Requires-Dist: datachain[audio,hf,remote,torch,vector,video]; extra == "tests"
 Requires-Dist: pytest<9,>=8; extra == "tests"
 Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
 Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
@@ -108,6 +111,7 @@ Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: ultralytics; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
+Requires-Dist: openai; extra == "examples"
 Dynamic: license-file
 
 ================

{datachain-0.25.1 → datachain-0.26.0}/docs/commands/job/run.md

@@ -31,6 +31,7 @@ This command runs a job in Studio using the specified query file. You can config
 * `--req-file REQ_FILE` - Python requirements file
 * `--req REQ` - Python package requirements
 * `--priority PRIORITY` - Priority for the job in range 0-5. Lower value is higher priority (default: 5)
+* `--repository URL` - Repository URL to clone before running the job.
 * `-h`, `--help` - Show the help message and exit.
 * `-v`, `--verbose` - Be verbose.
 * `-q`, `--quiet` - Be quiet.
@@ -67,6 +68,18 @@ datachain job run --env API_KEY=123 --req pandas numpy query.py
 datachain job run --repository https://github.com/iterative/datachain query.py
 ```
 
+To specify a branch / revision:
+
+```bash
+datachain job run --repository https://github.com/iterative/datachain@main query.py
+```
+
+Git URLs are also supported:
+
+```bash
+datachain job run --repository git@github.com:iterative/datachain.git@main query.py
+```
+
 7. Run a job with higher priority
 ```bash
 datachain job run --priority 2 query.py

{datachain-0.25.1 → datachain-0.26.0}/docs/references/datachain.md

@@ -15,6 +15,10 @@ for examples of how to create a chain.
 
 ::: datachain.lib.dc.datasets.datasets
 
+::: datachain.lib.dc.datasets.delete_dataset
+
+::: datachain.lib.dc.datasets.move_dataset
+
 ::: datachain.lib.dc.hf.read_hf
 
 ::: datachain.lib.dc.json.read_json

{datachain-0.25.1 → datachain-0.26.0}/examples/computer_vision/iptc_exif_xmp_lib.py

@@ -5,8 +5,6 @@ To install the required dependencies:
 
 """
 
-import json
-
 from PIL import (
     ExifTags,
     IptcImagePlugin,
@@ -14,8 +12,7 @@ from PIL import (
 )
 
 import datachain as dc
-
-source = "gs://datachain-demo/open-images-v6/"
+from datachain import C, DataModel, File
 
 
 def cast(v):  # to JSON serializable types
@@ -34,16 +31,21 @@ def cast(v):  # to JSON serializable types
     return v
 
 
-def image_description(file):
-    (xmp, exif, iptc) = ({}, {}, {})
+class ImageDescription(DataModel):
+    xmp: dict
+    exif: dict
+    iptc: dict
+
+
+def image_description(file: File) -> tuple[ImageDescription, str]:
+    xmp, exif, iptc = {}, {}, {}
     try:
         img = file.read()
         xmp = img.getxmp()
         img_exif = img.getexif()
         img_iptc = IptcImagePlugin.getiptcinfo(img)
     except Exception as err:  # noqa: BLE001
-        error = str(err)
-        return ({}, {}, {}, error)
+        return ImageDescription(xmp={}, exif={}, iptc={}), str(err)
 
     if img_iptc:
         for k, v in img_iptc.items():
@@ -57,26 +59,20 @@ def image_description(file):
             if k in ExifTags.GPSTAGS:
                 exif[ExifTags.GPSTAGS[k]] = v
 
-    return (
-        json.dumps(xmp),
-        json.dumps(exif),
-        json.dumps(iptc),
-        "",
-    )
+    return (ImageDescription(xmp=xmp, exif=exif, iptc=iptc), "")
 
 
 if __name__ == "__main__":
     (
-        dc.read_storage(source, type="image")
-        .settings(parallel=-1)
-        .filter(dc.C("file.path").glob("*.jpg"))
+        dc.read_storage("gs://datachain-demo/open-images-v6/", type="image", anon=True)
+        .filter(C("file.path").glob("*.jpg"))
         .limit(5000)
-        .map(
-            image_description,
-            params=["file"],
-            output={"xmp": dict, "exif": dict, "iptc": dict, "error": str},
+        .settings(parallel=True)
+        .map(image_description, output=("description", "error"))
+        .filter(
+            (C("description.xmp") != "{}")
+            | (C("description.exif") != "{}")
+            | (C("description.iptc") != "{}")
         )
-        .select("file.path", "xmp", "exif", "iptc", "error")
-        .filter((dc.C("xmp") != "{}") | (dc.C("exif") != "{}") | (dc.C("iptc") != "{}"))
         .show()
     )

datachain-0.26.0/examples/computer_vision/llava2_image_desc_lib.py

@@ -0,0 +1,71 @@
+"""
+To install the required dependencies:
+
+  pip install datachain[examples]
+
+"""
+
+import torch
+from transformers import (
+    AutoProcessor,
+    LlavaForConditionalGeneration,
+)
+
+import datachain as dc
+from datachain import C, File
+
+model = "llava-hf/llava-1.5-7b-hf"
+
+# HuggingFace supports the following base models:
+#
+# "llava-hf/llava-1.5-7b-hf"
+# "llava-hf/llava-1.5-13b-hf"
+# "llava-hf/bakLlava-v1-hf"
+#
+# https://huggingface.co/llava-hf
+
+
+# Probably this code can be written with HF pipeline
+# but we keep it a bit more low-level for the sake of example.
+class LLaVaProcessor:
+    def __init__(self, model_name, max_tokens=300):
+        if torch.cuda.is_available():
+            self.device = "cuda"
+            self.dtype = torch.float16
+        else:
+            self.device = "cpu"
+            self.dtype = torch.float32
+
+        self.model_name = model_name
+        self.max_tokens = max_tokens
+        self.prompt = "USER: <image>\nDescribe this picture\nASSISTANT:"
+
+        self.processor = AutoProcessor.from_pretrained(self.model_name)
+        self.model = LlavaForConditionalGeneration.from_pretrained(
+            self.model_name, torch_dtype=self.dtype, low_cpu_mem_usage=True
+        ).to(self.device)
+
+
+def process(processor: LLaVaProcessor, file: File) -> tuple[str, str]:
+    inputs = processor.processor(
+        text=processor.prompt, images=file.read(), return_tensors="pt"
+    ).to(processor.device, processor.dtype)
+
+    generated_ids = processor.model.generate(
+        **inputs, max_new_tokens=processor.max_tokens
+    )
+    generated_text = processor.processor.batch_decode(
+        generated_ids, skip_special_tokens=True
+    )
+    desc = generated_text[0]
+    return desc.split("ASSISTANT:")[-1].strip(), ""
+
+
+if __name__ == "__main__":
+    (
+        dc.read_storage("gs://datachain-demo/dogs-and-cats/", type="image", anon=True)
+        .filter(C("file.path").glob("*/cat*.jpg"))
+        .setup(processor=lambda: LLaVaProcessor(model_name=model))
+        .map(process, output=("description", "error"))
+        .show(5)
+    )

{datachain-0.25.1 → datachain-0.26.0}/examples/computer_vision/openimage-detect.py

@@ -1,19 +1,23 @@
 import json
+from collections.abc import Iterator
 
 from PIL import Image
 
 import datachain as dc
-from datachain import File, model
+from datachain import C, File, model
 from datachain.func import path
 
 
-def openimage_detect(args):
-    if len(args) != 2:
+# Example showing extraction of bounding boxes from Open Images dataset
+# that comes as pairs of JPG and JSON files.
+def openimage_detect(file: list[File]) -> Iterator[tuple[File, model.BBox]]:
+    if len(file) != 2:
         raise ValueError("Group jpg-json mismatch")
 
-    stream_jpg = args[0]
-    stream_json = args[1]
-    if args[0].get_file_ext() != "jpg":
+    stream_jpg = file[0]
+    stream_json = file[1]
+    source = stream_jpg.source
+    if stream_jpg.get_file_ext() != "jpg":
         stream_jpg, stream_json = stream_json, stream_jpg
 
     with stream_jpg.open() as fd:
@@ -38,16 +42,14 @@ def openimage_detect(args):
         yield fstream, bbox
 
 
-source = "gs://datachain-demo/openimages-v6-test-jsonpairs/"
-
 (
-    dc.read_storage(source)
-    .filter(dc.C("file.path").glob("*.jpg") | dc.C("file.path").glob("*.json"))
+    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/", anon=True)
+    .filter(C("file.path").glob("*.jpg") | C("file.path").glob("*.json"))
+    .settings(cache=True, parallel=True)
     .agg(
         openimage_detect,
         partition_by=path.file_stem("file.path"),
-        params=["file"],
-        output={"file": File, "bbox": model.BBox},
+        output=("file", "bbox"),
     )
     .show()
 )

{datachain-0.25.1 → datachain-0.26.0}/examples/computer_vision/ultralytics-bbox.py

@@ -10,7 +10,7 @@ def process_bboxes(yolo: YOLO, file: dc.File) -> YoloBBoxes:
 
 
 (
-    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/", anon=True)
     .filter(dc.C("file.path").glob("*.jpg"))
     .limit(20)
     .setup(yolo=lambda: YOLO("yolo11n.pt"))

{datachain-0.25.1 → datachain-0.26.0}/examples/computer_vision/ultralytics-pose.py

@@ -10,7 +10,7 @@ def process_poses(yolo: YOLO, file: dc.File) -> YoloPoses:
 
 
 (
-    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/", anon=True)
     .filter(dc.C("file.path").glob("*.jpg"))
     .limit(20)
     .setup(yolo=lambda: YOLO("yolo11n-pose.pt"))

{datachain-0.25.1 → datachain-0.26.0}/examples/computer_vision/ultralytics-segment.py

@@ -10,7 +10,7 @@ def process_segments(yolo: YOLO, file: dc.File) -> YoloSegments:
 
 
 (
-    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/")
+    dc.read_storage("gs://datachain-demo/openimages-v6-test-jsonpairs/", anon=True)
     .filter(dc.C("file.path").glob("*.jpg"))
     .limit(20)
     .setup(yolo=lambda: YOLO("yolo11n-seg.pt"))

{datachain-0.25.1 → datachain-0.26.0}/examples/get_started/common_sql_functions.py

@@ -1,23 +1,19 @@
+"""
+Example demonstrating showing functions (manipulating strings, paths, arrays)
+that are translated directly to SQL (vectorized). They don't require heavy compute,
+fetching object into cluster, etc.
+"""
+
 import datachain as dc
+from datachain import C
 from datachain.func import array, greatest, least, path, string
 
-
-def num_chars_udf(file):
-    parts = file.name.split(".")
-    if len(parts) > 1:
-        return (list(parts[1]),)
-    return ([],)
-
-
 chain = dc.read_storage("gs://datachain-demo/dogs-and-cats/", anon=True)
-chain.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).select(
-    "file.path", "num_chars"
-).show(5)
 
 (
     chain.mutate(
-        length=string.length(path.name(dc.C("file.path"))),
-        parts=string.split(path.name(dc.C("file.path")), "."),
+        length=string.length(path.name(C("file.path"))),
+        parts=string.split(path.name(C("file.path")), "."),
     )
     .select("file.path", "length", "parts")
     .show(5)
@@ -25,14 +21,14 @@ chain.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).selec
 
 (
     chain.mutate(
-        stem=path.file_stem(dc.C("file.path")),
-        ext=path.file_ext(dc.C("file.path")),
+        stem=path.file_stem(C("file.path")),
+        ext=path.file_ext(C("file.path")),
     )
     .select("file.path", "stem", "ext")
     .show(5)
 )
 
-parts = string.split(path.name(dc.C("file.path")), ".")
+parts = string.split(path.name(C("file.path")), ".")
 chain = chain.mutate(
     isdog=array.contains(parts, "dog"),
     iscat=array.contains(parts, "cat"),
@@ -46,8 +42,8 @@ chain = chain.mutate(
 
 (
     chain.mutate(
-        greatest=greatest(chain.column("a"), dc.C("b")),
-        least=least(chain.column("a"), dc.C("b")),
+        greatest=greatest(chain.column("a"), C("b")),
+        least=least(chain.column("a"), C("b")),
     )
     .select("a", "b", "greatest", "least")
     .show(10)

{datachain-0.25.1 → datachain-0.26.0}/examples/get_started/json-csv-reader.py

@@ -1,33 +1,25 @@
 from typing import Optional
 
-from pydantic import BaseModel
-
 import datachain as dc
-from datachain.lib.data_model import ModelStore
+from datachain import DataModel
 from datachain.lib.meta_formats import gen_datamodel_code
 
 
 # Sample model for static JSON model
-class LicenseModel(BaseModel):
+class LicenseModel(DataModel):
     url: str
     id: int
     name: str
 
 
-LicenseFeature = ModelStore.register(LicenseModel)
-
-
 # Sample model for static CSV model
-class ChatDialog(BaseModel):
+class ChatDialog(DataModel):
     id: Optional[int] = None
     count: Optional[int] = None
     sender: Optional[str] = None
     text: Optional[str] = None
 
 
-ChatFeature = ModelStore.register(ChatDialog)
-
-
 def main():
     # Dynamic JSONl schema from 2 objects
     uri = "gs://datachain-demo/jsonl/object.jsonl"
@@ -53,7 +45,7 @@ def main():
 
     # Static JSON schema test parsing 3/7 objects
     static_json_ds = dc.read_json(
-        uri, jmespath="licenses", spec=LicenseFeature, nrows=3, anon="True"
+        uri, jmespath="licenses", spec=LicenseModel, nrows=3, anon="True"
     )
     static_json_ds.show()
 
@@ -73,6 +65,11 @@ def main():
     dynamic_csv_ds.print_schema()
     dynamic_csv_ds.show()
 
+    print(
+        "Note: script might hang at the end due to https://github.com/apache/arrow/issues/43497"
+    )
+    print("Just press Ctrl+C to exit.")
+
 
 if __name__ == "__main__":
     main()

{datachain-0.25.1 → datachain-0.26.0}/examples/get_started/torch-loader.py

@@ -55,8 +55,8 @@ class CNN(nn.Module):
 
 if __name__ == "__main__":
     ds = (
-        dc.read_storage(STORAGE, type="image")
-        .settings(prefetch=25)
+        dc.read_storage(STORAGE, type="image", anon=True)
+        .settings(prefetch=25, cache=True)
         .filter(dc.C("file.path").glob("*.jpg"))
         .map(
             label=lambda path: label_to_int(basename(path)[:3], CLASSES),

{datachain-0.25.1 → datachain-0.26.0}/examples/get_started/udfs/parallel.py

@@ -21,19 +21,19 @@ def fibonacci(n):
 
 
 # Define the UDF:
-def path_len_benchmark(path):
+def path_len_benchmark(path: str) -> int:
     # Run the fibonacci benchmark as an example of a single-threaded CPU-bound UDF
     fibonacci(35)
     if path.endswith(".json"):
-        return (-1,)
+        return -1
     return len(path)
 
 
 # Run in chain
-dc.read_storage(
-    "gs://datachain-demo/dogs-and-cats/",
-).settings(parallel=-1).map(
-    path_len_benchmark,
-    params=["file.path"],
-    output={"path_len": int},
-).show()
+(
+    dc.read_storage("gs://datachain-demo/dogs-and-cats/", anon=True)
+    # Try to disable to see the difference in performance
+    .settings(parallel=-1)
+    .map(path_len=path_len_benchmark, params=["file.path"])
+    .show()
+)

datachain-0.26.0/examples/get_started/udfs/simple.py

@@ -0,0 +1,21 @@
+import datachain as dc
+
+
+# Define the UDF:
+# DataChain figures out input and output types automatically
+# based on the function signature and the data provided.
+def path_len(path: str) -> int:
+    if path.endswith(".json"):
+        return -1
+    return len(path)
+
+
+if __name__ == "__main__":
+    # Process all the files in the storage bucket, using the UDF
+    # `read_storage` reads files from the specified path
+    # and returns a DataChain object that has `File` objects
+    (
+        dc.read_storage("gs://datachain-demo/dogs-and-cats/", anon=True)
+        .map(path_len=path_len, params=["file.path"])
+        .show()
+    )

{datachain-0.25.1 → datachain-0.26.0}/examples/get_started/udfs/stateful.py

@@ -7,41 +7,37 @@ To install the required dependencies:
 
 import os
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
-
 import open_clip
 
 import datachain as dc
+from datachain import C, File
+
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
 
 
-class ImageEncoder(dc.Mapper):
+class ClipImageEncoder:
     def __init__(self, model_name: str, pretrained: str):
         self.model_name = model_name
         self.pretrained = pretrained
-
-    def setup(self):
         self.model, _, self.preprocess = open_clip.create_model_and_transforms(
             self.model_name, self.pretrained
         )
 
-    def process(self, file) -> list[float]:
-        img = file.read()
-        img = self.preprocess(img).unsqueeze(0)
-        emb = self.model.encode_image(img)
-        return emb[0].tolist()
+
+def embeddings(file: File, encoder: ClipImageEncoder) -> list[float]:
+    img = file.read()
+    img = encoder.preprocess(img).unsqueeze(0)
+    emb = encoder.model.encode_image(img)
+    return emb[0].tolist()
 
 
 if __name__ == "__main__":
-    # Run in chain
     (
-        dc.read_storage("gs://datachain-demo/dogs-and-cats/", type="image")
-        .filter(dc.C("file.path").glob("*cat*.jpg"))
-        .settings(parallel=2)
+        dc.read_storage("gs://datachain-demo/dogs-and-cats/", type="image", anon=True)
+        .filter(C("file.path").glob("*cat*.jpg"))
         .limit(5)
-        .map(
-            ImageEncoder("ViT-B-32", "laion2b_s34b_b79k"),
-            params=["file"],
-            output={"emb": list[float]},
-        )
+        .settings(parallel=True)
+        .setup(encoder=lambda: ClipImageEncoder("ViT-B-32", "laion2b_s34b_b79k"))
+        .map(emb=embeddings)
         .show()
     )

datachain-0.26.0/examples/llm_and_nlp/claude-query.py

@@ -0,0 +1,65 @@
+import os
+import sys
+
+import anthropic
+from pydantic import BaseModel
+
+import datachain as dc
+from datachain import C, File
+
+DATA = "gs://datachain-demo/chatbot-KiT"
+MODEL = "claude-3-5-haiku-latest"
+TEMPERATURE = 0.9
+DEFAULT_OUTPUT_TOKENS = 1024
+
+PROMPT = """Consider the dialogue between the 'user' and the 'bot'. The 'user' is a
+ human trying to find the best mobile plan. The 'bot' is a chatbot designed to query
+ the user and offer the best  solution. The dialog is successful if the 'bot' is able to
+ gather the information and offer a plan, or inform the user that such plan does not
+ exist. The dialog is not successful if the conversation ends early or the 'user'
+ requests additional functions the 'bot' cannot perform. Read the dialogue below and
+ rate it 'Success' if it is successful, and 'Failure' if not. After that, provide
+ one-sentence explanation of the reasons for this rating. Use only JSON object as output
+ with the keys 'status', and 'explanation'.
+"""
+
+API_KEY = os.environ.get("ANTHROPIC_API_KEY")
+
+if not API_KEY:
+    print("This example requires an Anthropic API key")
+    print("Add your key using the ANTHROPIC_API_KEY environment variable.")
+    sys.exit(0)
+
+
+class Rating(BaseModel):
+    status: str = ""
+    explanation: str = ""
+
+
+def rate(client: anthropic.Anthropic, file: File) -> Rating:
+    content = file.read()
+    response = client.messages.create(
+        model=MODEL,
+        max_tokens=DEFAULT_OUTPUT_TOKENS,
+        system=PROMPT,
+        temperature=TEMPERATURE,
+        messages=[
+            {"role": "user", "content": f"{content}"},
+        ],
+    )
+
+    first_block = response.content[0]
+    if first_block.type == "text":
+        return Rating.model_validate_json(first_block.text)
+    raise ValueError(f"Unexpected content block type: {first_block.type}")
+
+
+(
+    dc.read_storage(DATA, type="text", anon=True)
+    .filter(C("file.path").glob("*.txt"))
+    .limit(4)
+    .settings(parallel=2, cache=True)
+    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
+    .map(rating=rate)
+    .show()
+)

{datachain-0.25.1 → datachain-0.26.0}/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -25,20 +25,20 @@ def eval_dialog(
 ) -> DialogEval:
     try:
         completion = client.chat_completion(
-            model="meta-llama/Llama-3.3-70B-Instruct",
+            model="HuggingFaceTB/SmolLM3-3B",
             messages=[
                 {
                     "role": "user",
                     "content": f"{PROMPT}\n\nUser: {user_input}\nBot: {bot_response}",
                 },
             ],
-            response_format={"type": "json", "value": DialogEval.model_json_schema()},
+            response_format={
+                "type": "json_schema",
+                "json_schema": {"schema": DialogEval.model_json_schema()},
+            },
         )
     except HTTPError as e:
-        return DialogEval(
-            result="Error",
-            reason=f"Error while interacting with the Hugging Face API. {e}",
-        )
+        return DialogEval(result="Error", reason=str(e))
 
     message = completion.choices[0].message
     try: