datachain 0.10.0__tar.gz → 0.11.0__tar.gz

{datachain-0.10.0 → datachain-0.11.0}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.9.6'
+    rev: 'v0.9.7'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.10.0 → datachain-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.10.0
+Version: 0.11.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -49,6 +49,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
 Requires-Dist: websockets
+Requires-Dist: tomli; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -102,7 +103,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.74; extra == "examples"
+Requires-Dist: ultralytics==8.3.78; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================

{datachain-0.10.0 → datachain-0.11.0}/pyproject.toml

@@ -51,7 +51,8 @@ dependencies = [
   "platformdirs",
   "dvc-studio-client>=0.21,<1",
   "tabulate",
-  "websockets"
+  "websockets",
+  "tomli;python_version<'3.11'"
 ]
 
 [project.optional-dependencies]
@@ -118,7 +119,7 @@ examples = [
   "defusedxml",
   "accelerate",
   "huggingface_hub[hf_transfer]",
-  "ultralytics==8.3.74",
+  "ultralytics==8.3.78",
   "open_clip_torch"
 ]
 

{datachain-0.10.0 → datachain-0.11.0}/src/datachain/lib/file.py

@@ -272,8 +272,12 @@ class File(DataModel):
     def save(self, destination: str):
         """Writes it's content to destination"""
         destination = stringify_path(destination)
-        client: Client = self._catalog.get_client(str(destination))
-        client.upload(self.read(), str(destination))
+        client: Client = self._catalog.get_client(destination)
+
+        if client.PREFIX == "file://" and not destination.startswith(client.PREFIX):
+            destination = Path(destination).absolute().as_uri()
+
+        client.upload(self.read(), destination)
 
     def _symlink_to(self, destination: str):
         if self.location:

datachain-0.11.0/src/datachain/script_meta.py

@@ -0,0 +1,147 @@
+import re
+from dataclasses import dataclass
+from typing import Any, Optional
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    # tomllib is in standard library from python 3.11 so for earlier versions
+    # we need tomli
+    import tomli as tomllib  # type: ignore[no-redef]
+
+
+class ScriptConfigParsingError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+@dataclass
+class ScriptConfig:
+    """
+    Class that is parsing inline script metadata to get some basic information for
+    running datachain script like python version, dependencies, attachments etc.
+    Inline script metadata must follow the format described in https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata.
+    Example of script with inline metadata:
+        # /// script
+        # requires-python = ">=3.12"
+        #
+        # dependencies = [
+        #   "pandas < 2.1.0",
+        #   "numpy == 1.26.4"
+        # ]
+        #
+        # [tools.datachain.workers]
+        # num_workers = 3
+        #
+        # [tools.datachain.attachments]
+        # image1 = "s3://ldb-public/image1.jpg"
+        # file1 = "s3://ldb-public/file.pdf"
+        #
+        # [tools.datachain.params]
+        # min_length_sec = 1
+        # cache = false
+        #
+        # [tools.datachain.inputs]
+        # threshold = 0.5
+        # start_ds_name = "ds://start"
+        #
+        # [tools.datachain.outputs]
+        # result_dataset = "ds://res"
+        # result_dir = "/temp"
+        #
+        # ///
+
+        import sys
+        import pandas as pd
+
+        print(f"Python version: {sys.version_info}")
+        print(f"Pandas version: {pd.__version__}")
+
+    """
+
+    python_version: Optional[str]
+    dependencies: list[str]
+    attachments: dict[str, str]
+    params: dict[str, Any]
+    inputs: dict[str, Any]
+    outputs: dict[str, Any]
+    num_workers: Optional[int] = None
+
+    def __init__(
+        self,
+        python_version: Optional[str] = None,
+        dependencies: Optional[list[str]] = None,
+        attachments: Optional[dict[str, str]] = None,
+        params: Optional[dict[str, Any]] = None,
+        inputs: Optional[dict[str, Any]] = None,
+        outputs: Optional[dict[str, Any]] = None,
+        num_workers: Optional[int] = None,
+    ):
+        self.python_version = python_version
+        self.dependencies = dependencies or []
+        self.attachments = attachments or {}
+        self.params = params or {}
+        self.inputs = inputs or {}
+        self.outputs = outputs or {}
+        self.num_workers = num_workers
+
+    def get_param(self, name: str, default: Any) -> Any:
+        return self.params.get(name, default)
+
+    def get_input(self, name: str, default: Any) -> Any:
+        return self.inputs.get(name, default)
+
+    def get_output(self, name: str, default: Any) -> Any:
+        return self.outputs.get(name, default)
+
+    def get_attachment(self, name: str, default: Any) -> Any:
+        return self.attachments.get(name, default)
+
+    @staticmethod
+    def read(script: str) -> Optional[dict]:
+        """Converts inline script metadata to dict with all found data"""
+        regex = (
+            r"(?m)^# \/\/\/ (?P<type>[a-zA-Z0-9-]+)[ \t]*$[\r\n|\r|\n]"
+            "(?P<content>(?:^#(?:| .*)$[\r\n|\r|\n])+)^# \\/\\/\\/[ \t]*$"
+        )
+        name = "script"
+        matches = list(
+            filter(lambda m: m.group("type") == name, re.finditer(regex, script))
+        )
+        if len(matches) > 1:
+            raise ValueError(f"Multiple {name} blocks found")
+        if len(matches) == 1:
+            content = "".join(
+                line[2:] if line.startswith("# ") else line[1:]
+                for line in matches[0].group("content").splitlines(keepends=True)
+            )
+            return tomllib.loads(content)
+        return None
+
+    @staticmethod
+    def parse(script: str) -> Optional["ScriptConfig"]:
+        """
+        Method that is parsing inline script metadata from datachain script and
+        instantiating ScriptConfig class with found data. If no inline metadata is
+        found, it returns None
+        """
+        try:
+            meta = ScriptConfig.read(script)
+            if not meta:
+                return None
+            custom = meta.get("tools", {}).get("datachain", {})
+            return ScriptConfig(
+                python_version=meta.get("requires-python"),
+                dependencies=meta.get("dependencies"),
+                num_workers=custom.get("workers", {}).get("num_workers"),
+                attachments=custom.get("attachments"),
+                params={k: str(v) for k, v in custom.get("params").items()}
+                if custom.get("params")
+                else None,
+                inputs=custom.get("inputs"),
+                outputs=custom.get("outputs"),
+            )
+        except Exception as e:
+            raise ScriptConfigParsingError(
+                f"Error when parsing script meta: {e}"
+            ) from e

{datachain-0.10.0 → datachain-0.11.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.10.0
+Version: 0.11.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -49,6 +49,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
 Requires-Dist: websockets
+Requires-Dist: tomli; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -102,7 +103,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.74; extra == "examples"
+Requires-Dist: ultralytics==8.3.78; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================

{datachain-0.10.0 → datachain-0.11.0}/src/datachain.egg-info/SOURCES.txt

@@ -77,6 +77,7 @@ src/datachain/nodes_fetcher.py
 src/datachain/nodes_thread_pool.py
 src/datachain/progress.py
 src/datachain/py.typed
+src/datachain/script_meta.py
 src/datachain/studio.py
 src/datachain/telemetry.py
 src/datachain/utils.py
@@ -279,6 +280,7 @@ tests/unit/test_pytorch.py
 tests/unit/test_query.py
 tests/unit/test_query_metrics.py
 tests/unit/test_query_params.py
+tests/unit/test_script_meta.py
 tests/unit/test_serializer.py
 tests/unit/test_session.py
 tests/unit/test_utils.py

{datachain-0.10.0 → datachain-0.11.0}/src/datachain.egg-info/requires.txt

@@ -32,6 +32,9 @@ dvc-studio-client<1,>=0.21
 tabulate
 websockets
 
+[:python_version < "3.11"]
+tomli
+
 [dev]
 datachain[docs,tests]
 mypy==1.15.0
@@ -55,7 +58,7 @@ datachain[tests]
 defusedxml
 accelerate
 huggingface_hub[hf_transfer]
-ultralytics==8.3.74
+ultralytics==8.3.78
 open_clip_torch
 
 [hf]

{datachain-0.10.0 → datachain-0.11.0}/tests/func/test_datachain.py

@@ -358,6 +358,27 @@ def test_export_images_files(test_session, tmp_dir, tmp_path, use_cache):
         assert images_equal(img["data"], exported_img)
 
 
+def test_to_storage_relative_path(test_session, tmp_path):
+    images = [
+        {"name": "img1.jpg", "data": Image.new(mode="RGB", size=(64, 64))},
+        {"name": "img2.jpg", "data": Image.new(mode="RGB", size=(128, 128))},
+    ]
+
+    for img in images:
+        img["data"].save(tmp_path / img["name"])
+
+    DataChain.from_values(
+        file=[
+            ImageFile(path=img["name"], source=f"file://{tmp_path}") for img in images
+        ],
+        session=test_session,
+    ).to_storage("output", placement="filename")
+
+    for img in images:
+        exported_img = Image.open(Path("output") / img["name"])
+        assert images_equal(img["data"], exported_img)
+
+
 def test_to_storage_files_filename_placement_not_unique_files(tmp_dir, test_session):
     data = b"some\x00data\x00is\x48\x65\x6c\x57\x6f\x72\x6c\x64\xff\xffheRe"
     bucket_name = "mybucket"

datachain-0.11.0/tests/unit/test_script_meta.py

@@ -0,0 +1,119 @@
+import pytest
+
+from datachain.script_meta import ScriptConfig, ScriptConfigParsingError
+
+
+def test_parsing_all_fields():
+    script = """
+# /// script
+# requires-python = ">=3.12"
+#
+# dependencies = [
+#   "pandas < 2.1.0",
+#   "numpy == 1.26.4"
+# ]
+#
+# [tools.datachain.workers]
+# num_workers = 3
+#
+# [tools.datachain.attachments]
+# image1 = "s3://ldb-public/image1.jpg"
+# file1 = "s3://ldb-public/file.pdf"
+#
+# [tools.datachain.params]
+# min_length_sec = 1
+# cache = false
+#
+# [tools.datachain.inputs]
+# threshold = 0.5
+# start_ds_name = "ds://start"
+#
+# [tools.datachain.outputs]
+# result_dataset = "ds://res"
+# result_dir = "/temp"
+#
+# ///
+import sys
+import pandas as pd
+
+print(f"Python version: {sys.version_info}")
+print(f"Pandas version: {pd.__version__}")
+"""
+    sm = ScriptConfig.parse(script)
+    assert sm == ScriptConfig(
+        python_version=">=3.12",
+        dependencies=["pandas < 2.1.0", "numpy == 1.26.4"],
+        attachments={
+            "image1": "s3://ldb-public/image1.jpg",
+            "file1": "s3://ldb-public/file.pdf",
+        },
+        params={"min_length_sec": "1", "cache": "False"},
+        inputs={"threshold": 0.5, "start_ds_name": "ds://start"},
+        outputs={"result_dataset": "ds://res", "result_dir": "/temp"},
+        num_workers=3,
+    )
+    assert sm.get_param("non_existing", "default") == "default"
+
+
+def test_parsing_no_metadata():
+    script = """
+import sys
+import pandas as pd
+
+print(f"Python version: {sys.version_info}")
+print(f"Pandas version: {pd.__version__}")
+"""
+
+    assert ScriptConfig.parse(script) is None
+
+
+def test_parsing_empty():
+    script = """
+# /// script
+# ///
+import sys
+import pandas as pd
+
+print(f"Python version: {sys.version_info}")
+print(f"Pandas version: {pd.__version__}")
+"""
+
+    assert ScriptConfig.parse(script) is None
+
+
+def test_parsing_only_python_version():
+    script = """
+# /// script
+# requires-python = ">=3.12"
+# ///
+import sys
+import pandas as pd
+
+print(f"Python version: {sys.version_info}")
+print(f"Pandas version: {pd.__version__}")
+"""
+    assert ScriptConfig.parse(script) == ScriptConfig(
+        python_version=">=3.12",
+        dependencies=[],
+        attachments={},
+        params={},
+        num_workers=None,
+    )
+
+
+def test_error_when_parsing():
+    script = """
+# /// script
+# dependencies = [}
+# ///
+import sys
+import pandas as pd
+
+print(f"Python version: {sys.version_info}")
+print(f"Pandas version: {pd.__version__}")
+"""
+    with pytest.raises(ScriptConfigParsingError) as excinfo:
+        ScriptConfig.parse(script)
+    assert str(excinfo.value) == (
+        "Error when parsing script meta: Invalid value (at line 1, column 17)"
+    )