datachain 0.6.3__tar.gz → 0.6.5__tar.gz

{datachain-0.6.3 → datachain-0.6.5}/.github/workflows/tests.yml

@@ -152,4 +152,6 @@ jobs:
         run: uv pip install nox --system
 
       - name: Run examples
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: nox -s examples -p ${{ matrix.pyv }} -- -m "${{ matrix.group }}"

{datachain-0.6.3/src/datachain.egg-info → datachain-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.3
+Version: 0.6.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

datachain-0.6.5/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -0,0 +1,59 @@
+from huggingface_hub import InferenceClient
+
+from datachain import C, DataChain, DataModel
+
+PROMPT = """
+Was this dialog successful? Put result as a single word: Success or Failure.
+Explain the reason in a few words.
+"""
+
+
+class DialogEval(DataModel):
+    result: str
+    reason: str
+
+
+# DataChain function to evaluate dialog.
+# DataChain is using types for inputs, results to automatically infer schema.
+def eval_dialog(user_input: str, bot_response: str) -> DialogEval:
+    client = InferenceClient("meta-llama/Llama-3.1-70B-Instruct")
+
+    completion = client.chat_completion(
+        messages=[
+            {
+                "role": "user",
+                "content": f"{PROMPT}\n\nUser: {user_input}\nBot: {bot_response}",
+            },
+        ],
+        response_format={"type": "json", "value": DialogEval.model_json_schema()},
+    )
+
+    message = completion.choices[0].message
+    try:
+        return DialogEval.model_validate_json(message.content)
+    except ValueError:
+        return DialogEval(result="Error", reason="Failed to parse response.")
+
+
+# Run HF inference in parallel for each example.
+# Get result as Pydantic model that DataChain can understand and serialize it.
+# Save to HF as Parquet. Dataset can be previewed here:
+# https://huggingface.co/datasets/dvcorg/test-datachain-llm-eval/viewer
+(
+    DataChain.from_csv(
+        "hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv"
+    )
+    .settings(parallel=10)
+    .map(response=eval_dialog)
+    .to_parquet("hf://datasets/dvcorg/test-datachain-llm-eval/data.parquet")
+)
+
+# Read it back to filter and show.
+# It restores the Pydantic model from Parquet under the hood.
+(
+    DataChain.from_parquet(
+        "hf://datasets/dvcorg/test-datachain-llm-eval/data.parquet", source=False
+    )
+    .filter(C("response.result") == "Failure")
+    .show(3)
+)

{datachain-0.6.3 → datachain-0.6.5}/src/datachain/catalog/catalog.py

@@ -42,6 +42,7 @@ from datachain.dataset import (
     DatasetStats,
     DatasetStatus,
     RowDict,
+    StorageURI,
     create_dataset_uri,
     parse_dataset_uri,
 )
@@ -58,7 +59,6 @@ from datachain.node import DirType, Node, NodeWithPath
 from datachain.nodes_thread_pool import NodesThreadPool
 from datachain.remote.studio import StudioClient
 from datachain.sql.types import DateTime, SQLType, String
-from datachain.storage import StorageURI
 from datachain.utils import (
     DataChainDir,
     batched,
@@ -1702,31 +1702,9 @@ class Catalog:
         *,
         client_config=None,
     ) -> None:
-        root_sources = [
-            src for src in sources if Client.get_implementation(src).is_root_url(src)
-        ]
-        non_root_sources = [
-            src
-            for src in sources
-            if not Client.get_implementation(src).is_root_url(src)
-        ]
-
-        client_config = client_config or self.client_config
-
-        # for root sources (e.g s3://) we are just getting all buckets and
-        # saving them as storages, without further indexing in each bucket
-        for source in root_sources:
-            for bucket in Client.get_implementation(source).ls_buckets(**client_config):
-                client = self.get_client(bucket.uri, **client_config)
-                print(f"Registering storage {client.uri}")
-                self.metastore.create_storage_if_not_registered(client.uri)
-
         self.enlist_sources(
-            non_root_sources,
+            sources,
             update,
-            client_config=client_config,
+            client_config=client_config or self.client_config,
             only_index=True,
         )
-
-    def find_stale_storages(self) -> None:
-        self.metastore.find_stale_storages()

{datachain-0.6.3 → datachain-0.6.5}/src/datachain/cli.py

@@ -568,12 +568,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
     add_sources_arg(parse_index)
 
-    subp.add_parser(
-        "find-stale-storages",
-        parents=[parent_parser],
-        description="Finds and marks stale storages",
-    )
-
     show_parser = subp.add_parser(
         "show",
         parents=[parent_parser],
@@ -1100,8 +1094,6 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
             )
         elif args.command == "completion":
             print(completion(args.shell))
-        elif args.command == "find-stale-storages":
-            catalog.find_stale_storages()
         elif args.command == "query":
             query(
                 catalog,

{datachain-0.6.3 → datachain-0.6.5}/src/datachain/client/fsspec.py

@@ -31,11 +31,12 @@ from datachain.error import ClientError as DataChainClientError
 from datachain.lib.file import File
 from datachain.nodes_fetcher import NodesFetcher
 from datachain.nodes_thread_pool import NodeChunk
-from datachain.storage import StorageURI
 
 if TYPE_CHECKING:
     from fsspec.spec import AbstractFileSystem
 
+    from datachain.dataset import StorageURI
+
 
 logger = logging.getLogger("datachain")
 
@@ -63,7 +64,7 @@ def _is_win_local_path(uri: str) -> bool:
 
 class Bucket(NamedTuple):
     name: str
-    uri: StorageURI
+    uri: "StorageURI"
     created: Optional[datetime]
 
 
@@ -115,7 +116,7 @@ class Client(ABC):
         return DATA_SOURCE_URI_PATTERN.match(name) is not None
 
     @staticmethod
-    def parse_url(source: str) -> tuple[StorageURI, str]:
+    def parse_url(source: str) -> tuple["StorageURI", str]:
         cls = Client.get_implementation(source)
         storage_name, rel_path = cls.split_url(source)
         return cls.get_uri(storage_name), rel_path
@@ -148,7 +149,7 @@ class Client(ABC):
     @classmethod
     def from_source(
         cls,
-        uri: StorageURI,
+        uri: "StorageURI",
         cache: DataChainCache,
         **kwargs,
     ) -> "Client":
@@ -156,6 +157,8 @@ class Client(ABC):
 
     @classmethod
     def ls_buckets(cls, **kwargs) -> Iterator[Bucket]:
+        from datachain.dataset import StorageURI
+
         for entry in cls.create_fs(**kwargs).ls(cls.PREFIX, detail=True):
             name = entry["name"].rstrip("/")
             yield Bucket(
@@ -169,7 +172,9 @@ class Client(ABC):
         return url == cls.PREFIX
 
     @classmethod
-    def get_uri(cls, name) -> StorageURI:
+    def get_uri(cls, name) -> "StorageURI":
+        from datachain.dataset import StorageURI
+
         return StorageURI(f"{cls.PREFIX}{name}")
 
     @classmethod

{datachain-0.6.3 → datachain-0.6.5}/src/datachain/client/hf.py

@@ -23,6 +23,7 @@ class HfClient(Client):
 
     def info_to_file(self, v: dict[str, Any], path: str) -> File:
         return File(
+            source=self.uri,
             path=path,
             size=v["size"],
             version=v["last_commit"].oid,

{datachain-0.6.3 → datachain-0.6.5}/src/datachain/client/local.py

@@ -2,16 +2,18 @@ import os
 import posixpath
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 from fsspec.implementations.local import LocalFileSystem
 
 from datachain.lib.file import File
-from datachain.storage import StorageURI
 
 from .fsspec import Client
 
+if TYPE_CHECKING:
+    from datachain.dataset import StorageURI
+
 
 class FileClient(Client):
     FS_CLASS = LocalFileSystem
@@ -28,7 +30,9 @@ class FileClient(Client):
         raise TypeError("Signed urls are not implemented for local file system")
 
     @classmethod
-    def get_uri(cls, name) -> StorageURI:
+    def get_uri(cls, name) -> "StorageURI":
+        from datachain.dataset import StorageURI
+
         return StorageURI(f'{cls.PREFIX}/{name.removeprefix("/")}')
 
     @classmethod