datachain 0.8.10__tar.gz → 0.8.12__tar.gz

{datachain-0.8.10 → datachain-0.8.12}/.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: Tests
 on:
   push:
     branches: [main]
-  pull_request_target:
+  pull_request:
   workflow_dispatch:
 
 env:
@@ -14,15 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  authorize:
-    environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }}
-    runs-on: ubuntu-latest
-    steps:
-      - run: true
-
   lint:
-    needs: authorize
-
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
@@ -62,8 +54,6 @@ jobs:
         run: nox -s lint
 
   datachain:
-    needs: authorize
-
     timeout-minutes: 40
     runs-on: ${{ matrix.os }}
     strategy:
@@ -112,7 +102,11 @@ jobs:
         run: echo 'DISABLE_REMOTES_ARG=--disable-remotes=azure,gs' >> $env:GITHUB_ENV
 
       - name: Run tests
-        run: nox -s tests-${{ matrix.pyv }} -- $DISABLE_REMOTES_ARG
+        run: nox -s tests-${{ matrix.pyv }} -- -m "not e2e and not examples" $DISABLE_REMOTES_ARG
+        shell: bash
+
+      - name: Run E2E tests
+        run: nox -s tests-${{ matrix.pyv }} -- -m "e2e" --cov-append $DISABLE_REMOTES_ARG
         shell: bash
 
       - name: Upload coverage report
@@ -129,8 +123,6 @@ jobs:
         run: nox -s docs
 
   examples:
-    needs: authorize
-
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     strategy:

{datachain-0.8.10 → datachain-0.8.12}/.pre-commit-config.yaml

@@ -24,13 +24,13 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.9.1'
+    rev: 'v0.9.3'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.0
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]

{datachain-0.8.10 → datachain-0.8.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.10
+Version: 0.8.12
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -78,7 +78,6 @@ Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
 Requires-Dist: virtualenv; extra == "tests"
 Requires-Dist: dulwich; extra == "tests"
 Requires-Dist: hypothesis; extra == "tests"
-Requires-Dist: open_clip_torch; extra == "tests"
 Requires-Dist: aiotools>=1.7.0; extra == "tests"
 Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
@@ -94,12 +93,9 @@ Provides-Extra: examples
 Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
-Requires-Dist: unstructured_ingest[embed-huggingface]; extra == "examples"
-Requires-Dist: unstructured[pdf]<0.16.12; extra == "examples"
-Requires-Dist: pdfplumber==0.11.5; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.61; extra == "examples"
+Requires-Dist: ultralytics==8.3.68; extra == "examples"
+Requires-Dist: open_clip_torch; extra == "examples"
 
 ================
 |logo| DataChain

{datachain-0.8.10 → datachain-0.8.12}/docs/overrides/main.html

@@ -8,6 +8,16 @@
 <script type="text/javascript">
   !function () { var e, t, n; e = "14ffd92a6cbf5f2", t = function () { Reo.init({ clientID: "14ffd92a6cbf5f2" }) }, (n = document.createElement("script")).src = "https://static.reo.dev/" + e + "/reo.js", n.async = !0, n.onload = t, document.head.appendChild(n) }();
 </script>
+<script>
+  function initApollo() {
+    var n = Math.random().toString(36).substring(7), o = document.createElement("script");
+    o.src = "https://assets.apollo.io/micro/website-tracker/tracker.iife.js?nocache=" + n, o.async = !0, o.defer = !0,
+      o.onload = function () { window.trackingFunctions.onLoad({ appId: "66315101e9aa7501c79140d9" }) },
+      document.head.appendChild(o)
+  };
+  initApollo();
+</script>
+
 
 
 {% endblock %}

datachain-0.8.12/docs/references/func.md

@@ -0,0 +1,5 @@
+# Functions
+
+Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate). Import these functions from `datachain.func`.
+
+::: datachain.func

{datachain-0.8.10 → datachain-0.8.12}/docs/references/index.md

@@ -10,5 +10,5 @@ DataChain's API is organized into several modules:
 - [DataType](./datatype.md) - Type system and schema definitions
 - [File](./file.md) - File handling and storage operations
 - [UDF](./udf.md) - User-defined functions and transformations
-- [SQL](./sql.md) - SQL query integration
+- [Functions](./func.md) - Built-in functions for data manipulation and analysis
 - [Torch](./torch.md) - PyTorch data loading utilities

{datachain-0.8.10 → datachain-0.8.12}/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -1,4 +1,5 @@
 from huggingface_hub import InferenceClient
+from requests import HTTPError
 
 from datachain import C, DataChain, DataModel
 
@@ -20,15 +21,20 @@ def eval_dialog(
     user_input: str,
     bot_response: str,
 ) -> DialogEval:
-    completion = client.chat_completion(
-        messages=[
-            {
-                "role": "user",
-                "content": f"{PROMPT}\n\nUser: {user_input}\nBot: {bot_response}",
-            },
-        ],
-        response_format={"type": "json", "value": DialogEval.model_json_schema()},
-    )
+    try:
+        completion = client.chat_completion(
+            messages=[
+                {
+                    "role": "user",
+                    "content": f"{PROMPT}\n\nUser: {user_input}\nBot: {bot_response}",
+                },
+            ],
+            response_format={"type": "json", "value": DialogEval.model_json_schema()},
+        )
+    except HTTPError:
+        return DialogEval(
+            result="Error", reason="Error while interacting with the Hugging Face API."
+        )
 
     message = completion.choices[0].message
     try:

{datachain-0.8.10 → datachain-0.8.12}/mkdocs.yml

@@ -73,7 +73,7 @@ nav:
           - File: references/file.md
           - UDF: references/udf.md
           - Torch: references/torch.md
-          - SQL: references/sql.md
+          - Functions: references/func.md
       - 🤝 Contributing: contributing.md
 
   - DataChain Website ↗: https://datachain.ai" target="_blank"

{datachain-0.8.10 → datachain-0.8.12}/pyproject.toml

@@ -89,7 +89,6 @@ tests = [
   "virtualenv",
   "dulwich",
   "hypothesis",
-  "open_clip_torch",
   "aiotools>=1.7.0",
   "requests-mock",
   "scipy"
@@ -107,12 +106,9 @@ examples = [
   "datachain[tests]",
   "defusedxml",
   "accelerate",
-  "unstructured_ingest[embed-huggingface]",
-  "unstructured[pdf]<0.16.12",
-  "pdfplumber==0.11.5",
   "huggingface_hub[hf_transfer]",
-  "onnx==1.16.1",
-  "ultralytics==8.3.61"
+  "ultralytics==8.3.68",
+  "open_clip_torch"
 ]
 
 [project.urls]

{datachain-0.8.10 → datachain-0.8.12}/src/datachain/cache.py

@@ -22,15 +22,15 @@ def try_scandir(path):
         pass
 
 
-def get_temp_cache(tmp_dir: str, prefix: Optional[str] = None) -> "DataChainCache":
+def get_temp_cache(tmp_dir: str, prefix: Optional[str] = None) -> "Cache":
     cache_dir = mkdtemp(prefix=prefix, dir=tmp_dir)
-    return DataChainCache(cache_dir, tmp_dir=tmp_dir)
+    return Cache(cache_dir, tmp_dir=tmp_dir)
 
 
 @contextmanager
 def temporary_cache(
     tmp_dir: str, prefix: Optional[str] = None, delete: bool = True
-) -> Iterator["DataChainCache"]:
+) -> Iterator["Cache"]:
     cache = get_temp_cache(tmp_dir, prefix=prefix)
     try:
         yield cache
@@ -39,7 +39,7 @@ def temporary_cache(
             cache.destroy()
 
 
-class DataChainCache:
+class Cache:
     def __init__(self, cache_dir: str, tmp_dir: str):
         self.odb = LocalHashFileDB(
             LocalFileSystem(),

{datachain-0.8.10 → datachain-0.8.12}/src/datachain/catalog/__init__.py

@@ -3,7 +3,6 @@ from .catalog import (
     QUERY_SCRIPT_CANCELED_EXIT_CODE,
     QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE,
     Catalog,
-    parse_edatachain_file,
 )
 from .loader import get_catalog
 
@@ -13,5 +12,4 @@ __all__ = [
     "QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE",
     "Catalog",
     "get_catalog",
-    "parse_edatachain_file",
 ]