datachain 0.7.10__tar.gz → 0.8.0__tar.gz

{datachain-0.7.10 → datachain-0.8.0}/.github/workflows/tests.yml

@@ -136,7 +136,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, windows-latest]
         pyv: ['3.9', '3.12']
         group: ['get_started', 'llm_and_nlp or computer_vision', 'multimodal']
         exclude:
@@ -166,7 +166,20 @@ jobs:
       - name: Install nox
         run: uv pip install nox --system
 
+      # HF runs against actual API - thus run it only once
+      - name: Set hf token
+        if: matrix.os == 'ubuntu-latest' && matrix.pyv == '3.12'
+        run: echo 'HF_TOKEN=${{ secrets.HF_TOKEN }}' >> "$GITHUB_ENV"
+
       - name: Run examples
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: nox -s examples -p ${{ matrix.pyv }} -- -m "${{ matrix.group }}"
+
+  check:
+    if: always()
+    needs: [lint, datachain, examples]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: re-actors/alls-green@release/v1
+        with:
+          allowed-failures: examples
+          jobs: ${{ toJSON(needs) }}

{datachain-0.7.10 → datachain-0.8.0}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.8.1'
+    rev: 'v0.8.3'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.7.10/src/datachain.egg-info → datachain-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.10
+Version: 0.8.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -46,6 +46,7 @@ Requires-Dist: iterative-telemetry>=0.0.9
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
+Requires-Dist: websockets
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -91,14 +92,14 @@ Requires-Dist: types-requests; extra == "dev"
 Requires-Dist: types-tabulate; extra == "dev"
 Provides-Extra: examples
 Requires-Dist: datachain[tests]; extra == "examples"
-Requires-Dist: numpy<2,>=1; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
-Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
+Requires-Dist: unstructured_ingest[embed-huggingface]; extra == "examples"
+Requires-Dist: unstructured[pdf]; extra == "examples"
 Requires-Dist: pdfplumber==0.11.4; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.37; extra == "examples"
+Requires-Dist: ultralytics==8.3.50; extra == "examples"
 
 ================
 |logo| DataChain
@@ -138,6 +139,11 @@ Use Cases
 3. **Versioning.** DataChain doesn't store, require moving or copying data (unlike DVC).
    Perfect use case is a bucket with thousands or millions of images, videos, audio, PDFs.
 
+Getting Started
+===============
+
+Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ and `Docs <https://docs.datachain.ai/>`_
+to get started with `DataChain` and learn more.
 
 Key Features
 ============
@@ -161,12 +167,6 @@ Key Features
    - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
 
-Getting Started
-===============
-
-Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ to get started with `DataChain` and learn more.
-
-
 Contributing
 ============
 

{datachain-0.7.10 → datachain-0.8.0}/README.rst

@@ -36,6 +36,11 @@ Use Cases
 3. **Versioning.** DataChain doesn't store, require moving or copying data (unlike DVC).
    Perfect use case is a bucket with thousands or millions of images, videos, audio, PDFs.
 
+Getting Started
+===============
+
+Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ and `Docs <https://docs.datachain.ai/>`_
+to get started with `DataChain` and learn more.
 
 Key Features
 ============
@@ -59,12 +64,6 @@ Key Features
    - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
 
-Getting Started
-===============
-
-Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ to get started with `DataChain` and learn more.
-
-
 Contributing
 ============
 

{datachain-0.7.10 → datachain-0.8.0}/docs/contributing.md

@@ -1,3 +1,7 @@
+---
+title: Contributing
+---
+
 # Contributor Guide
 
 Thank you for your interest in improving this project. This project is

datachain-0.8.0/docs/css/github-permalink-style.css

@@ -0,0 +1,39 @@
+.headerlink {
+	--permalink-size: 16px; /* for font-relative sizes, 0.6em is a good choice */
+	--permalink-spacing: 4px;
+
+	width: calc(var(--permalink-size) + var(--permalink-spacing));
+	height: var(--permalink-size);
+	vertical-align: middle;
+	background-color: var(--md-default-fg-color--lighter);
+	background-size: var(--permalink-size);
+	mask-size: var(--permalink-size);
+	-webkit-mask-size: var(--permalink-size);
+	mask-repeat: no-repeat;
+	-webkit-mask-repeat: no-repeat;
+	visibility: visible;
+	mask-image: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="16" height="16" aria-hidden="true"><path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"></path></svg>');
+	-webkit-mask-image: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="16" height="16" aria-hidden="true"><path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"></path></svg>');
+}
+
+[id]:target .headerlink {
+	background-color: var(--md-typeset-a-color);
+}
+
+.headerlink:hover {
+	background-color: var(--md-accent-fg-color) !important;
+}
+
+@media screen and (min-width: 76.25em) {
+	h1, h2, h3, h4, h5, h6 {
+		display: flex;
+		align-items: center;
+		flex-direction: row;
+		column-gap: 0.2em; /* fixes spaces in titles */
+	}
+
+	.headerlink {
+		order: -1;
+		margin-left: calc(var(--permalink-size) * -1 - var(--permalink-spacing)) !important;
+	}
+}

{datachain-0.7.10 → datachain-0.8.0}/docs/examples.md

@@ -1,3 +1,6 @@
+---
+title: Examples
+---
 
 # Examples
 
@@ -225,7 +228,7 @@ Here is an example from MS COCO “captions” JSON which employs separate secti
 }
 ```
 
-Note how complicated the setup is. Every image is references by the name, and the metadata for this file is keyed by the “id” field. This same field is references later in the “annotations’ array, which is present in JSON files describing captions and the detected instances. The categories for the instances are stored in the “categories” array.
+Note how complicated the setup is. Every image is references by the name, and the metadata for this file is keyed by the “id” field. This same field is references later in the “annotations” array, which is present in JSON files describing captions and the detected instances. The categories for the instances are stored in the “categories” array.
 
 However, Datachain can easily parse the entire COCO structure via several reading and merging operators:
 

{datachain-0.7.10 → datachain-0.8.0}/docs/index.md

@@ -1,3 +1,6 @@
+---
+title: Welcome to DataChain
+---
 # <a class="main-header-link" href="/" ><img style="display: inline-block;" src="/assets/datachain.svg" alt="DataChain"> <span style="display: inline-block;"> DataChain</span></a>
 
 <style>
@@ -83,7 +86,7 @@ The following pages provide detailed documentation on DataChain's features, arch
 - [🏃🏼‍♂️ Quick Start](quick-start.md): Get up and running with DataChain in no time.
 - [🎯 Examples](examples.md): Explore practical examples and use cases.
 - [📚 Tutorials](tutorials.md): Learn how to use DataChain for specific tasks.
-- [📚 API Reference](references/index.md): Dive into the technical details and API reference.
+- [🐍 API Reference](references/index.md): Dive into the technical details and API reference.
 - [🤝 Contributing](contributing.md): Learn how to contribute to DataChain.
 
 

{datachain-0.7.10 → datachain-0.8.0}/docs/quick-start.md

@@ -1,3 +1,7 @@
+---
+title: Quick Start
+---
+
 # Quick Start
 
 ## Installation

{datachain-0.7.10 → datachain-0.8.0}/docs/references/index.md

@@ -1,3 +1,7 @@
+---
+title: API Reference
+---
+
 # API Reference
 
 DataChain's API is organized into several modules:

{datachain-0.7.10 → datachain-0.8.0}/docs/tutorials.md

@@ -1,3 +1,7 @@
+---
+title: Tutorials
+---
+
 # Tutorials
 
 * Multimodal: [GitHub](https://github.com/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb) or [Google Colab](https://colab.research.google.com/github/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb)

datachain-0.8.0/examples/get_started/json-csv-reader.py

@@ -0,0 +1,82 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from datachain import C, DataChain
+from datachain.lib.data_model import ModelStore
+from datachain.lib.meta_formats import gen_datamodel_code
+
+
+# Sample model for static JSON model
+class LicenseModel(BaseModel):
+    url: str
+    id: int
+    name: str
+
+
+LicenseFeature = ModelStore.register(LicenseModel)
+
+
+# Sample model for static CSV model
+class ChatDialog(BaseModel):
+    id: Optional[int] = None
+    count: Optional[int] = None
+    sender: Optional[str] = None
+    text: Optional[str] = None
+
+
+ChatFeature = ModelStore.register(ChatDialog)
+
+
+def main():
+    # Dynamic JSONl schema from 2 objects
+    uri = "gs://datachain-demo/jsonl/object.jsonl"
+    jsonl_ds = DataChain.from_json(uri, format="jsonl", anon="True")
+    jsonl_ds.show()
+
+    # Dynamic JSON schema from 200 OpenImage json-pairs with validation errors
+    uri = "gs://datachain-demo/openimages-v6-test-jsonpairs/*json"
+    schema_uri = (
+        "gs://datachain-demo/openimages-v6-test-jsonpairs/08392c290ecc9d2a.json"
+    )
+    json_pairs_ds = DataChain.from_json(
+        uri, schema_from=schema_uri, jmespath="@", model_name="OpenImage", anon="True"
+    )
+    json_pairs_ds.show()
+
+    uri = "gs://datachain-demo/coco2017/annotations_captions/"
+
+    # Print JSON schema in Pydantic format from main COCO annotation
+    chain = DataChain.from_storage(uri, anon="True").filter(
+        C("file.path").glob("*.json")
+    )
+    file = next(chain.limit(1).collect("file"))
+    print(gen_datamodel_code(file, jmespath="@", model_name="Coco"))
+
+    # Static JSON schema test parsing 3/7 objects
+    static_json_ds = DataChain.from_json(
+        uri, jmespath="licenses", spec=LicenseFeature, nrows=3, anon="True"
+    )
+    static_json_ds.show()
+
+    # Dynamic JSON schema test parsing 5K objects
+    dynamic_json_ds = DataChain.from_json(uri, jmespath="images", anon="True")
+    print(dynamic_json_ds.to_pandas())
+
+    # Static CSV with header schema test parsing 3.5K objects
+    uri = "gs://datachain-demo/chatbot-csv/"
+    static_csv_ds = DataChain.from_csv(
+        uri, output=ChatDialog, object_name="chat", anon="True"
+    )
+    static_csv_ds.print_schema()
+    static_csv_ds.show()
+
+    # Dynamic CSV with header schema test parsing 3/3M objects
+    uri = "gs://datachain-demo/laion-aesthetics-csv/laion_aesthetics_1024_33M_1.csv"
+    dynamic_csv_ds = DataChain.from_csv(uri, object_name="laion", nrows=3, anon="True")
+    dynamic_csv_ds.print_schema()
+    dynamic_csv_ds.show()
+
+
+if __name__ == "__main__":
+    main()

{datachain-0.7.10 → datachain-0.8.0}/examples/get_started/torch-loader.py

@@ -5,6 +5,7 @@ To install the required dependencies:
 
 """
 
+import multiprocessing
 import os
 from posixpath import basename
 
@@ -12,17 +13,18 @@ import torch
 from torch import nn, optim
 from torch.utils.data import DataLoader
 from torchvision.transforms import v2
+from tqdm import tqdm
 
 from datachain import C, DataChain
 from datachain.torch import label_to_int
 
 STORAGE = "gs://datachain-demo/dogs-and-cats/"
-NUM_EPOCHS = os.getenv("NUM_EPOCHS", "3")
+NUM_EPOCHS = int(os.getenv("NUM_EPOCHS", "3"))
 
 # Define transformation for data preprocessing
 transform = v2.Compose(
     [
-        v2.ToTensor(),
+        v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]),
         v2.Resize((64, 64)),
         v2.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
     ]
@@ -54,6 +56,7 @@ class CNN(nn.Module):
 if __name__ == "__main__":
     ds = (
         DataChain.from_storage(STORAGE, type="image")
+        .settings(cache=True, prefetch=25)
         .filter(C("file.path").glob("*.jpg"))
         .map(
             label=lambda path: label_to_int(basename(path)[:3], CLASSES),
@@ -64,8 +67,10 @@ if __name__ == "__main__":
 
     train_loader = DataLoader(
         ds.to_pytorch(transform=transform),
-        batch_size=16,
-        num_workers=2,
+        batch_size=25,
+        num_workers=max(4, os.cpu_count() or 2),
+        persistent_workers=True,
+        multiprocessing_context=multiprocessing.get_context("spawn"),
     )
 
     model = CNN()
@@ -73,19 +78,19 @@ if __name__ == "__main__":
     optimizer = optim.Adam(model.parameters(), lr=0.001)
 
     # Train the model
-    for epoch in range(int(NUM_EPOCHS)):
-        for i, data in enumerate(train_loader):
-            inputs, labels = data
-            optimizer.zero_grad()
-
-            # Forward pass
-            outputs = model(inputs)
-            loss = criterion(outputs, labels)
-
-            # Backward pass and optimize
-            loss.backward()
-            optimizer.step()
-
-            print(f"[{epoch + 1}, {i + 1:5d}] loss: {loss.item():.3f}")
-
-    print("Finished Training")
+    for epoch in range(NUM_EPOCHS):
+        with tqdm(
+            train_loader, desc=f"epoch {epoch + 1}/{NUM_EPOCHS}", unit="batch"
+        ) as loader:
+            for data in loader:
+                inputs, labels = data
+                optimizer.zero_grad()
+
+                # Forward pass
+                outputs = model(inputs)
+                loss = criterion(outputs, labels)
+
+                # Backward pass and optimize
+                loss.backward()
+                optimizer.step()
+                loader.set_postfix(loss=loss.item())

{datachain-0.7.10 → datachain-0.8.0}/examples/llm_and_nlp/unstructured-embeddings-gen.py

@@ -12,11 +12,11 @@ from unstructured.cleaners.core import (
     group_broken_paragraphs,
     replace_unicode_quotes,
 )
-from unstructured.embed.huggingface import (
+from unstructured.partition.pdf import partition_pdf
+from unstructured_ingest.embed.huggingface import (
     HuggingFaceEmbeddingConfig,
     HuggingFaceEmbeddingEncoder,
 )
-from unstructured.partition.pdf import partition_pdf
 
 from datachain import C, DataChain, DataModel, File
 
@@ -43,6 +43,7 @@ def process_pdf(file: File) -> Iterator[Chunk]:
         chunks = partition_pdf(file=f, chunking_strategy="by_title", strategy="fast")
 
     # Clean the chunks and add new columns
+    text_chunks = []
     for chunk in chunks:
         chunk.apply(
             lambda text: clean(
@@ -51,16 +52,17 @@ def process_pdf(file: File) -> Iterator[Chunk]:
         )
         chunk.apply(replace_unicode_quotes)
         chunk.apply(group_broken_paragraphs)
+        text_chunks.append({"text": str(chunk)})
 
     # create embeddings
-    chunks_embedded = embedding_encoder.embed_documents(chunks)
+    chunks_embedded = embedding_encoder.embed_documents(text_chunks)
 
     # Add new rows to DataChain
     for chunk in chunks_embedded:
         yield Chunk(
             key=file.path,
-            text=chunk.text,
-            embeddings=chunk.embeddings,
+            text=chunk.get("text"),
+            embeddings=chunk.get("embeddings"),
         )
 
 

{datachain-0.7.10 → datachain-0.8.0}/mkdocs.yml

@@ -27,7 +27,6 @@ theme:
     - navigation.tabs
     - navigation.path
     - navigation.top
-    - navigation.prune
     - navigation.footer
     - toc.follow
     - content.action.edit
@@ -37,7 +36,6 @@ theme:
     - content.tooltips
     - search.highlight
     - search.suggest
-    - navigation.sections
 
   palette:
     # Palette toggle for automatic mode
@@ -56,8 +54,8 @@ theme:
     # Palette toggle for dark mode
     - media: "(prefers-color-scheme: dark)"
       scheme: slate
-      primary: black
-      accent: lime
+      primary: teal
+      accent: teal
       toggle:
         icon: material/weather-night
         name: Switch to system preference
@@ -68,18 +66,18 @@ nav:
       - 🏃🏼‍♂️ Quick Start: quick-start.md
       - 🎯 Examples: examples.md
       - 📚 Tutorials: tutorials.md
-      - 🐍 API Reference: references/index.md
+      - 🐍 API Reference:
+          - Overview: references/index.md
+          - DataChain: references/datachain.md
+          - DataType: references/datatype.md
+          - File: references/file.md
+          - UDF: references/udf.md
+          - Torch: references/torch.md
+          - SQL: references/sql.md
       - 🤝 Contributing: contributing.md
-  - API Reference:
-      - references/index.md
-      - references/datachain.md
-      - references/datatype.md
-      - references/file.md
-      - references/udf.md
-      - references/torch.md
-      - references/sql.md
-  - DataChain Website: https://datachain.ai" target="_blank"
-  - Studio: https://studio.datachain.ai" target="_blank"
+
+  - DataChain Website ↗: https://datachain.ai" target="_blank"
+  - Studio ↗: https://studio.datachain.ai" target="_blank"
 
 markdown_extensions:
   - abbr
@@ -105,7 +103,11 @@ markdown_extensions:
   - pymdownx.tilde
   - tables
   - toc:
-      permalink: true
+      permalink: ''
+
+# Custom permalink style: https://github.com/squidfunk/mkdocs-material/discussions/3535
+extra_css:
+  - css/github-permalink-style.css
 
 extra:
   social:

{datachain-0.7.10 → datachain-0.8.0}/noxfile.py

@@ -81,6 +81,8 @@ def examples(session: nox.Session) -> None:
     session.install(".[examples]")
     session.run(
         "pytest",
+        "--durations=0",
+        "tests/examples",
         "-m",
         "examples",
         *session.posargs,

{datachain-0.7.10 → datachain-0.8.0}/pyproject.toml

@@ -48,7 +48,8 @@ dependencies = [
   "iterative-telemetry>=0.0.9",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",
-  "tabulate"
+  "tabulate",
+  "websockets"
 ]
 
 [project.optional-dependencies]
@@ -104,14 +105,14 @@ dev = [
 ]
 examples = [
   "datachain[tests]",
-  "numpy>=1,<2",
   "defusedxml",
   "accelerate",
-  "unstructured[pdf,embed-huggingface]<0.16.0",
+  "unstructured_ingest[embed-huggingface]",
+  "unstructured[pdf]",
   "pdfplumber==0.11.4",
   "huggingface_hub[hf_transfer]",
   "onnx==1.16.1",
-  "ultralytics==8.3.37"
+  "ultralytics==8.3.50"
 ]
 
 [project.urls]