datachain 0.2.9__tar.gz → 0.2.11__tar.gz

{datachain-0.2.9 → datachain-0.2.11}/.github/workflows/tests.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Lint code
         run: nox -s lint
 
-  tests:
+  datachain:
     timeout-minutes: 25
     runs-on: ${{ matrix.os }}
     strategy:
@@ -125,3 +125,73 @@ jobs:
 
       - name: Build docs
         run: nox -s docs
+
+
+  studio:
+    if: '!github.event.pull_request.head.repo.fork'
+    runs-on: ubuntu-latest-16-cores
+    strategy:
+      matrix:
+        pyv: ['3.12']
+        group: [1, 2, 3, 4, 5, 6]
+    services:
+      postgres:
+        image: postgres:16.3
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: test
+          POSTGRES_DB: database
+          POSTGRES_HOST_AUTH_METHOD: trust
+      clickhouse:
+        image: clickhouse/clickhouse-server:24
+        ports:
+          - 8123:8123
+          - 9010:9000
+        env:
+          CLICKHOUSE_DB: studio_local_db
+          CLICKHOUSE_USER: studio_local
+          CLICKHOUSE_PASSWORD: ch123456789!
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+      redis:
+        image: redis:7.2.5
+        ports:
+          - 6379:6379
+    steps:
+
+      - name: Check out Studio
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          repository: iterative/studio
+          ref: develop
+          token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          path: './backend/datachain'
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.pyv }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pyv }}
+          cache: 'pip'
+
+      - name: Install uv
+        run: |
+          python -m pip install --upgrade uv
+          uv --version
+
+      - name: Install dependencies
+        run: uv pip install --system ./backend/datachain_server[tests] ./backend/datachain[tests]
+
+      - name: Run tests
+        # Generate `.test_durations` file with `pytest --store-durations --durations-path ../.github/.test_durations ...`
+        run: >
+          pytest
+          --config-file=pyproject.toml -rsx
+          --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
+          tests ../datachain/tests
+        working-directory: backend/datachain_server

{datachain-0.2.9/src/datachain.egg-info → datachain-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.9
+Version: 0.2.11
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -35,10 +35,12 @@ Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: dill==0.3.8
+Requires-Dist: cloudpickle
 Requires-Dist: ujson>=5.9.0
 Requires-Dist: pydantic<3,>=2
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
+Requires-Dist: Pillow<11,>=10.0.0
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -46,11 +48,10 @@ Requires-Dist: mkdocs-material>=9.3.1; extra == "docs"
 Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
 Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
 Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
-Provides-Extra: cv
-Requires-Dist: Pillow<11,>=10.0.0; extra == "cv"
-Requires-Dist: torch>=2.1.0; extra == "cv"
-Requires-Dist: torchvision; extra == "cv"
-Requires-Dist: transformers>=4.36.0; extra == "cv"
+Provides-Extra: torch
+Requires-Dist: torch>=2.1.0; extra == "torch"
+Requires-Dist: torchvision; extra == "torch"
+Requires-Dist: transformers>=4.36.0; extra == "torch"
 Provides-Extra: remote
 Requires-Dist: lz4; extra == "remote"
 Requires-Dist: msgpack<2,>=1.0.4; extra == "remote"
@@ -58,7 +59,7 @@ Requires-Dist: requests>=2.22.0; extra == "remote"
 Provides-Extra: vector
 Requires-Dist: usearch; extra == "vector"
 Provides-Extra: tests
-Requires-Dist: datachain[cv,remote,vector]; extra == "tests"
+Requires-Dist: datachain[remote,torch,vector]; extra == "tests"
 Requires-Dist: pytest<9,>=8; extra == "tests"
 Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
 Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
@@ -89,11 +90,11 @@ Requires-Dist: types-ujson; extra == "dev"
 .. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
    :target: https://pypi.org/project/datachain
    :alt: Python Version
-.. |Codecov| image:: https://codecov.io/gh/iterative/dvcx/branch/main/graph/badge.svg?token=VSCP2T9R5X
-   :target: https://app.codecov.io/gh/iterative/dvcx
+.. |Codecov| image:: https://codecov.io/gh/iterative/datachain/graph/badge.svg?token=byliXGGyGB
+   :target: https://codecov.io/gh/iterative/datachain
    :alt: Codecov
-.. |Tests| image:: https://github.com/iterative/dvcx/workflows/Tests/badge.svg
-   :target: https://github.com/iterative/dvcx/actions?workflow=Tests
+.. |Tests| image:: https://github.com/iterative/datachain/workflows/Tests/badge.svg
+   :target: https://github.com/iterative/datachain/actions?workflow=Tests
    :alt: Tests
 
 AI 🔗 DataChain
@@ -397,7 +398,8 @@ Chain results can be exported or passed directly to Pytorch dataloader. For exam
 Tutorials
 ------------------
 
-* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/dvclive/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
+* `Computer Vision <examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain/blob/main/examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`__)
+* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
 
 Contributions
 --------------------

{datachain-0.2.9 → datachain-0.2.11}/README.rst

@@ -6,11 +6,11 @@
 .. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
    :target: https://pypi.org/project/datachain
    :alt: Python Version
-.. |Codecov| image:: https://codecov.io/gh/iterative/dvcx/branch/main/graph/badge.svg?token=VSCP2T9R5X
-   :target: https://app.codecov.io/gh/iterative/dvcx
+.. |Codecov| image:: https://codecov.io/gh/iterative/datachain/graph/badge.svg?token=byliXGGyGB
+   :target: https://codecov.io/gh/iterative/datachain
    :alt: Codecov
-.. |Tests| image:: https://github.com/iterative/dvcx/workflows/Tests/badge.svg
-   :target: https://github.com/iterative/dvcx/actions?workflow=Tests
+.. |Tests| image:: https://github.com/iterative/datachain/workflows/Tests/badge.svg
+   :target: https://github.com/iterative/datachain/actions?workflow=Tests
    :alt: Tests
 
 AI 🔗 DataChain
@@ -314,7 +314,8 @@ Chain results can be exported or passed directly to Pytorch dataloader. For exam
 Tutorials
 ------------------
 
-* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/dvclive/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
+* `Computer Vision <examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain/blob/main/examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`__)
+* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
 
 Contributions
 --------------------

{datachain-0.2.9 → datachain-0.2.11}/examples/json-csv-reader.py

@@ -103,7 +103,8 @@ def main():
     print("========================================================================")
     print("static CSV with header schema test parsing 3.5K objects")
     print("========================================================================")
-    static_csv_ds = DataChain.from_csv(uri, spec=ChatFeature)
+    static_csv_ds = DataChain.from_csv(uri, output=ChatFeature, object_name="chat")
+    static_csv_ds.print_schema()
     print(static_csv_ds.to_pandas())
 
     uri = "gs://datachain-demo/laion-aesthetics-csv"
@@ -111,7 +112,8 @@ def main():
     print("========================================================================")
     print("dynamic CSV with header schema test parsing 3M objects")
     print("========================================================================")
-    dynamic_csv_ds = DataChain.from_csv(uri, object_name="laion", show_schema=True)
+    dynamic_csv_ds = DataChain.from_csv(uri, object_name="laion")
+    dynamic_csv_ds.print_schema()
     print(dynamic_csv_ds.to_pandas())
 
 

datachain-0.2.11/examples/llm-claude-aggregate-query.py

@@ -0,0 +1,57 @@
+import os
+
+import anthropic
+from anthropic.types import Message
+
+from datachain import Column, DataChain
+from datachain.sql.functions import path
+
+DATA = "gs://dvcx-datalakes/chatbot-public"
+MODEL = "claude-3-opus-20240229"
+PROMPT = """Consider the following dialogues between the 'user' and the 'bot' separated\
+ by '===='. The 'user' is a human trying to find the best mobile plan. The 'bot' is a \
+chatbot designed to query the user and offer the best solution. The dialog is \
+successful if the 'bot' is able to gather the information and offer a plan, or inform \
+the user that such plan does not exist. The dialog is not successful if the \
+conversation ends early or the 'user' requests additional functions the 'bot' \
+cannot perform. Read the dialogues and classify them into a fixed number of concise \
+failure reasons covering most failure cases. Present output as JSON list of reason \
+strings and nothing else.
+"""
+
+TEMPERATURE = 0.9
+DEFAULT_OUTPUT_TOKENS = 1024
+
+API_KEY = os.environ.get("ANTHROPIC_API_KEY")
+
+
+chain = (
+    DataChain.from_storage(DATA, type="text")
+    .filter(Column("file.name").glob("*.txt"))
+    .limit(5)
+    .settings(parallel=4, cache=True)
+    .agg(
+        dialogues=lambda file: ["\n=====\n".join(f.read() for f in file)],
+        output=str,
+        partition_by=path.file_ext(Column("name")),
+    )
+    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
+    .map(
+        claude=lambda client, dialogues: client.messages.create(
+            model=MODEL,
+            system=PROMPT,
+            messages=[
+                {"role": "user", "content": dialogues},
+            ],
+            temperature=TEMPERATURE,
+            max_tokens=DEFAULT_OUTPUT_TOKENS,
+        ),
+        output=Message,
+    )
+    .map(
+        res=lambda claude: claude.content[0].text if claude.content else [],
+        output=str,
+    )
+)
+
+chain.show()

{datachain-0.2.9 → datachain-0.2.11}/examples/llm-claude-simple-query.py

@@ -1,12 +1,13 @@
 import json
+import os
 
-import pandas as pd
+import anthropic
+from anthropic.types import Message
+from pydantic import BaseModel
 
-from datachain.lib.claude import claude_processor
-from datachain.lib.dc import C, DataChain
-from datachain.lib.feature import Feature
+from datachain import Column, DataChain, File
 
-SOURCE = "gs://dvcx-datalakes/chatbot-public"
+DATA = "gs://dvcx-datalakes/chatbot-public"
 MODEL = "claude-3-opus-20240229"
 PROMPT = """Consider the dialogue between the 'user' and the 'bot'. \
 The 'user' is a human trying to find the best mobile plan. \
@@ -20,19 +21,38 @@ if it is successful, and 'Failure' if not. After that, provide \
 one-sentence explanation of the reasons for this rating. Use only \
 JSON object as output with the keys 'status', and 'explanation'.
 """
+TEMPERATURE = 0.9
+DEFAULT_OUTPUT_TOKENS = 1024
 
+API_KEY = os.environ.get("ANTHROPIC_API_KEY")
 
-class Rating(Feature):
+
+class Rating(BaseModel):
     status: str = ""
     explanation: str = ""
 
 
 chain = (
-    DataChain.from_storage(SOURCE, type="text")
-    .filter(C.name.glob("*.txt"))
-    .settings(parallel=3)
+    DataChain.from_storage(DATA, type="text")
+    .filter(Column("file.name").glob("*.txt"))
     .limit(5)
-    .map(claude=claude_processor(prompt=PROMPT, model=MODEL))
+    .settings(parallel=4, cache=True)
+    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
+    .map(
+        claude=lambda client, file: client.messages.create(
+            model=MODEL,
+            system=PROMPT,
+            messages=[
+                {
+                    "role": "user",
+                    "content": file.read() if isinstance(file, File) else file,
+                },
+            ],
+            temperature=TEMPERATURE,
+            max_tokens=DEFAULT_OUTPUT_TOKENS,
+        ),
+        output=Message,
+    )
     .map(
         rating=lambda claude: Rating(
             **(json.loads(claude.content[0].text) if claude.content else {})
@@ -41,7 +61,4 @@ chain = (
     )
 )
 
-df = chain.to_pandas()
-
-with pd.option_context("display.max_columns", None):
-    print(df)
+chain.show()

datachain-0.2.11/examples/llm-claude.py

@@ -0,0 +1,39 @@
+import os
+
+import anthropic
+from anthropic.types import Message
+
+from datachain import Column, DataChain, File
+
+DATA = "gs://dvcx-datalakes/chatbot-public"
+MODEL = "claude-3-opus-20240229"
+PROMPT = """Summarise the dialog in a sentence"""
+TEMPERATURE = 0.9
+DEFAULT_OUTPUT_TOKENS = 1024
+
+API_KEY = os.environ.get("ANTHROPIC_API_KEY")
+
+chain = (
+    DataChain.from_storage(DATA, type="text")
+    .filter(Column("file.name").glob("*.txt"))
+    .limit(5)
+    .settings(parallel=4, cache=True)
+    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
+    .map(
+        claude=lambda client, file: client.messages.create(
+            model=MODEL,
+            system=PROMPT,
+            messages=[
+                {
+                    "role": "user",
+                    "content": file.read() if isinstance(file, File) else file,
+                },
+            ],
+            temperature=TEMPERATURE,
+            max_tokens=DEFAULT_OUTPUT_TOKENS,
+        ),
+        output=Message,
+    )
+)
+
+chain.show()