datachain 0.8.1__tar.gz → 0.8.2__tar.gz

{datachain-0.8.1/src/datachain.egg-info → datachain-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.8.1
+Version: 0.8.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -145,6 +145,88 @@ Getting Started
 Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ and `Docs <https://docs.datachain.ai/>`_
 to get started with `DataChain` and learn more.
 
+.. code:: bash
+
+        pip install datachain
+
+
+Example: download subset of files based on metadata
+---------------------------------------------------
+
+Sometimes users only need to download a specific subset of files from cloud storage,
+rather than the entire dataset.
+For example, you could use a JSON file's metadata to download just cat images with
+high confidence scores.
+
+
+.. code:: py
+
+    from datachain import Column, DataChain
+
+    meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
+
+    images_id = images.map(id=lambda file: file.path.split('.')[-2])
+    annotated = images_id.merge(meta, on="id", right_on="meta.id")
+
+    likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
+                                   & (Column("meta.inference.class_") == "cat"))
+    likely_cats.export_files("high-confidence-cats/", signal="file")
+
+
+Example: LLM based text-file evaluation
+---------------------------------------
+
+In this example, we evaluate chatbot conversations stored in text files
+using LLM based evaluation.
+
+.. code:: shell
+
+    $ pip install mistralai # Requires version >=1.0.0
+    $ export MISTRAL_API_KEY=_your_key_
+
+Python code:
+
+.. code:: py
+
+    from mistralai import Mistral
+    from datachain import File, DataChain, Column
+
+    PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
+
+    def eval_dialogue(file: File) -> bool:
+         client = Mistral()
+         response = client.chat.complete(
+             model="open-mixtral-8x22b",
+             messages=[{"role": "system", "content": PROMPT},
+                       {"role": "user", "content": file.read()}])
+         result = response.choices[0].message.content
+         return result.lower().startswith("success")
+
+    chain = (
+       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       .settings(parallel=4, cache=True)
+       .map(is_success=eval_dialogue)
+       .save("mistral_files")
+    )
+
+    successful_chain = chain.filter(Column("is_success") == True)
+    successful_chain.export_files("./output_mistral")
+
+    print(f"{successful_chain.count()} files were exported")
+
+
+
+With the instruction above, the Mistral model considers 31/50 files to hold the successful dialogues:
+
+.. code:: shell
+
+    $ ls output_mistral/datachain-demo/chatbot-KiT/
+    1.txt  15.txt 18.txt 2.txt  22.txt 25.txt 28.txt 33.txt 37.txt 4.txt  41.txt ...
+    $ ls output_mistral/datachain-demo/chatbot-KiT/ | wc -l
+    31
+
+
 Key Features
 ============
 

{datachain-0.8.1 → datachain-0.8.2}/README.rst

@@ -42,6 +42,88 @@ Getting Started
 Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ and `Docs <https://docs.datachain.ai/>`_
 to get started with `DataChain` and learn more.
 
+.. code:: bash
+
+        pip install datachain
+
+
+Example: download subset of files based on metadata
+---------------------------------------------------
+
+Sometimes users only need to download a specific subset of files from cloud storage,
+rather than the entire dataset.
+For example, you could use a JSON file's metadata to download just cat images with
+high confidence scores.
+
+
+.. code:: py
+
+    from datachain import Column, DataChain
+
+    meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
+
+    images_id = images.map(id=lambda file: file.path.split('.')[-2])
+    annotated = images_id.merge(meta, on="id", right_on="meta.id")
+
+    likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
+                                   & (Column("meta.inference.class_") == "cat"))
+    likely_cats.export_files("high-confidence-cats/", signal="file")
+
+
+Example: LLM based text-file evaluation
+---------------------------------------
+
+In this example, we evaluate chatbot conversations stored in text files
+using LLM based evaluation.
+
+.. code:: shell
+
+    $ pip install mistralai # Requires version >=1.0.0
+    $ export MISTRAL_API_KEY=_your_key_
+
+Python code:
+
+.. code:: py
+
+    from mistralai import Mistral
+    from datachain import File, DataChain, Column
+
+    PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
+
+    def eval_dialogue(file: File) -> bool:
+         client = Mistral()
+         response = client.chat.complete(
+             model="open-mixtral-8x22b",
+             messages=[{"role": "system", "content": PROMPT},
+                       {"role": "user", "content": file.read()}])
+         result = response.choices[0].message.content
+         return result.lower().startswith("success")
+
+    chain = (
+       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       .settings(parallel=4, cache=True)
+       .map(is_success=eval_dialogue)
+       .save("mistral_files")
+    )
+
+    successful_chain = chain.filter(Column("is_success") == True)
+    successful_chain.export_files("./output_mistral")
+
+    print(f"{successful_chain.count()} files were exported")
+
+
+
+With the instruction above, the Mistral model considers 31/50 files to hold the successful dialogues:
+
+.. code:: shell
+
+    $ ls output_mistral/datachain-demo/chatbot-KiT/
+    1.txt  15.txt 18.txt 2.txt  22.txt 25.txt 28.txt 33.txt 37.txt 4.txt  41.txt ...
+    $ ls output_mistral/datachain-demo/chatbot-KiT/ | wc -l
+    31
+
+
 Key Features
 ============
 

{datachain-0.8.1 → datachain-0.8.2}/docs/quick-start.md

@@ -39,8 +39,8 @@ using JSON metadata:
 ``` py
 from datachain import Column, DataChain
 
-meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta")
-images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg")
+meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
 
 images_id = images.map(id=lambda file: file.path.split('.')[-2])
 annotated = images_id.merge(meta, on="id", right_on="meta.id")
@@ -78,7 +78,7 @@ def is_positive_dialogue_ending(file) -> bool:
 
 chain = (
    DataChain.from_storage("gs://datachain-demo/chatbot-KiT/",
-                          object_name="file", type="text")
+                          object_name="file", type="text", anon=True)
    .settings(parallel=8, cache=True)
    .map(is_positive=is_positive_dialogue_ending)
    .save("file_response")
@@ -132,7 +132,7 @@ def eval_dialogue(file: File) -> bool:
      return result.lower().startswith("success")
 
 chain = (
-   DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file")
+   DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
    .map(is_success=eval_dialogue)
    .save("mistral_files")
 )
@@ -177,7 +177,7 @@ def eval_dialog(file: File) -> ChatCompletionResponse:
                    {"role": "user", "content": file.read()}])
 
 chain = (
-   DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file")
+   DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
    .settings(parallel=4, cache=True)
    .map(response=eval_dialog)
    .map(status=lambda response: response.choices[0].message.content.lower()[:7])
@@ -273,7 +273,7 @@ from datachain import C, DataChain
 processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
 
 chain = (
-    DataChain.from_storage("gs://datachain-demo/dogs-and-cats/", type="image")
+    DataChain.from_storage("gs://datachain-demo/dogs-and-cats/", type="image", anon=True)
     .map(label=lambda name: name.split(".")[0], params=["file.name"])
     .select("file", "label").to_pytorch(
         transform=processor.image_processor,

{datachain-0.8.1 → datachain-0.8.2}/src/datachain/client/gcs.py

@@ -33,13 +33,14 @@ class GCSClient(Client):
         return cast(GCSFileSystem, super().create_fs(**kwargs))
 
     def url(self, path: str, expires: int = 3600, **kwargs) -> str:
-        try:
-            return self.fs.sign(self.get_full_path(path), expiration=expires, **kwargs)
-        except AttributeError as exc:
-            is_anon = self.fs.storage_options.get("token") == "anon"
-            if is_anon and "you need a private key to sign credentials" in str(exc):
-                return f"https://storage.googleapis.com/{self.name}/{path}"
-            raise
+        """
+        Generate a signed URL for the given path.
+        If the client is anonymous, a public URL is returned instead
+        (see https://cloud.google.com/storage/docs/access-public-data#api-link).
+        """
+        if self.fs.storage_options.get("token") == "anon":
+            return f"https://storage.googleapis.com/{self.name}/{path}"
+        return self.fs.sign(self.get_full_path(path), expiration=expires, **kwargs)
 
     @staticmethod
     def parse_timestamp(timestamp: str) -> datetime:

{datachain-0.8.1 → datachain-0.8.2/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.8.1
+Version: 0.8.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -145,6 +145,88 @@ Getting Started
 Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ and `Docs <https://docs.datachain.ai/>`_
 to get started with `DataChain` and learn more.
 
+.. code:: bash
+
+        pip install datachain
+
+
+Example: download subset of files based on metadata
+---------------------------------------------------
+
+Sometimes users only need to download a specific subset of files from cloud storage,
+rather than the entire dataset.
+For example, you could use a JSON file's metadata to download just cat images with
+high confidence scores.
+
+
+.. code:: py
+
+    from datachain import Column, DataChain
+
+    meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
+
+    images_id = images.map(id=lambda file: file.path.split('.')[-2])
+    annotated = images_id.merge(meta, on="id", right_on="meta.id")
+
+    likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
+                                   & (Column("meta.inference.class_") == "cat"))
+    likely_cats.export_files("high-confidence-cats/", signal="file")
+
+
+Example: LLM based text-file evaluation
+---------------------------------------
+
+In this example, we evaluate chatbot conversations stored in text files
+using LLM based evaluation.
+
+.. code:: shell
+
+    $ pip install mistralai # Requires version >=1.0.0
+    $ export MISTRAL_API_KEY=_your_key_
+
+Python code:
+
+.. code:: py
+
+    from mistralai import Mistral
+    from datachain import File, DataChain, Column
+
+    PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
+
+    def eval_dialogue(file: File) -> bool:
+         client = Mistral()
+         response = client.chat.complete(
+             model="open-mixtral-8x22b",
+             messages=[{"role": "system", "content": PROMPT},
+                       {"role": "user", "content": file.read()}])
+         result = response.choices[0].message.content
+         return result.lower().startswith("success")
+
+    chain = (
+       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       .settings(parallel=4, cache=True)
+       .map(is_success=eval_dialogue)
+       .save("mistral_files")
+    )
+
+    successful_chain = chain.filter(Column("is_success") == True)
+    successful_chain.export_files("./output_mistral")
+
+    print(f"{successful_chain.count()} files were exported")
+
+
+
+With the instruction above, the Mistral model considers 31/50 files to hold the successful dialogues:
+
+.. code:: shell
+
+    $ ls output_mistral/datachain-demo/chatbot-KiT/
+    1.txt  15.txt 18.txt 2.txt  22.txt 25.txt 28.txt 33.txt 37.txt 4.txt  41.txt ...
+    $ ls output_mistral/datachain-demo/chatbot-KiT/ | wc -l
+    31
+
+
 Key Features
 ============
 

datachain-0.8.2/tests/unit/test_client_gcs.py

@@ -0,0 +1,6 @@
+from datachain.client import Client
+
+
+def test_anon_url():
+    client = Client.get_client("gs://foo", None, anon=True)
+    assert client.url("bar") == "https://storage.googleapis.com/foo/bar"

datachain-0.8.1/tests/unit/test_client_gcs.py

@@ -1,17 +0,0 @@
-from datachain.client import Client
-
-
-def test_anon_url(mocker):
-    def sign(*args, **kwargs):
-        raise AttributeError(
-            "you need a private key to sign credentials."
-            "the credentials you are currently using"
-            " <class 'google.oauth2.credentials.Credentials'> just contains a token."
-            " see https://googleapis.dev/python/google-api-core/latest/auth.html"
-            "#setting-up-a-service-account for more details."
-        )
-
-    mocker.patch("gcsfs.GCSFileSystem.sign", side_effect=sign)
-
-    client = Client.get_client("gs://foo", None, anon=True)
-    assert client.url("bar") == "https://storage.googleapis.com/foo/bar"