datachain 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

datachain/lib/file.py

@@ -272,8 +272,12 @@ class File(DataModel):
     def save(self, destination: str):
         """Writes it's content to destination"""
         destination = stringify_path(destination)
-        client: Client = self._catalog.get_client(str(destination))
-        client.upload(self.read(), str(destination))
+        client: Client = self._catalog.get_client(destination)
+
+        if client.PREFIX == "file://" and not destination.startswith(client.PREFIX):
+            destination = Path(destination).absolute().as_uri()
+
+        client.upload(self.read(), destination)
 
     def _symlink_to(self, destination: str):
         if self.location:

datachain/script_meta.py

@@ -0,0 +1,147 @@
+import re
+from dataclasses import dataclass
+from typing import Any, Optional
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    # tomllib is in standard library from python 3.11 so for earlier versions
+    # we need tomli
+    import tomli as tomllib  # type: ignore[no-redef]
+
+
+class ScriptConfigParsingError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+@dataclass
+class ScriptConfig:
+    """
+    Class that is parsing inline script metadata to get some basic information for
+    running datachain script like python version, dependencies, attachments etc.
+    Inline script metadata must follow the format described in https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata.
+    Example of script with inline metadata:
+        # /// script
+        # requires-python = ">=3.12"
+        #
+        # dependencies = [
+        #   "pandas < 2.1.0",
+        #   "numpy == 1.26.4"
+        # ]
+        #
+        # [tools.datachain.workers]
+        # num_workers = 3
+        #
+        # [tools.datachain.attachments]
+        # image1 = "s3://ldb-public/image1.jpg"
+        # file1 = "s3://ldb-public/file.pdf"
+        #
+        # [tools.datachain.params]
+        # min_length_sec = 1
+        # cache = false
+        #
+        # [tools.datachain.inputs]
+        # threshold = 0.5
+        # start_ds_name = "ds://start"
+        #
+        # [tools.datachain.outputs]
+        # result_dataset = "ds://res"
+        # result_dir = "/temp"
+        #
+        # ///
+
+        import sys
+        import pandas as pd
+
+        print(f"Python version: {sys.version_info}")
+        print(f"Pandas version: {pd.__version__}")
+
+    """
+
+    python_version: Optional[str]
+    dependencies: list[str]
+    attachments: dict[str, str]
+    params: dict[str, Any]
+    inputs: dict[str, Any]
+    outputs: dict[str, Any]
+    num_workers: Optional[int] = None
+
+    def __init__(
+        self,
+        python_version: Optional[str] = None,
+        dependencies: Optional[list[str]] = None,
+        attachments: Optional[dict[str, str]] = None,
+        params: Optional[dict[str, Any]] = None,
+        inputs: Optional[dict[str, Any]] = None,
+        outputs: Optional[dict[str, Any]] = None,
+        num_workers: Optional[int] = None,
+    ):
+        self.python_version = python_version
+        self.dependencies = dependencies or []
+        self.attachments = attachments or {}
+        self.params = params or {}
+        self.inputs = inputs or {}
+        self.outputs = outputs or {}
+        self.num_workers = num_workers
+
+    def get_param(self, name: str, default: Any) -> Any:
+        return self.params.get(name, default)
+
+    def get_input(self, name: str, default: Any) -> Any:
+        return self.inputs.get(name, default)
+
+    def get_output(self, name: str, default: Any) -> Any:
+        return self.outputs.get(name, default)
+
+    def get_attachment(self, name: str, default: Any) -> Any:
+        return self.attachments.get(name, default)
+
+    @staticmethod
+    def read(script: str) -> Optional[dict]:
+        """Converts inline script metadata to dict with all found data"""
+        regex = (
+            r"(?m)^# \/\/\/ (?P<type>[a-zA-Z0-9-]+)[ \t]*$[\r\n|\r|\n]"
+            "(?P<content>(?:^#(?:| .*)$[\r\n|\r|\n])+)^# \\/\\/\\/[ \t]*$"
+        )
+        name = "script"
+        matches = list(
+            filter(lambda m: m.group("type") == name, re.finditer(regex, script))
+        )
+        if len(matches) > 1:
+            raise ValueError(f"Multiple {name} blocks found")
+        if len(matches) == 1:
+            content = "".join(
+                line[2:] if line.startswith("# ") else line[1:]
+                for line in matches[0].group("content").splitlines(keepends=True)
+            )
+            return tomllib.loads(content)
+        return None
+
+    @staticmethod
+    def parse(script: str) -> Optional["ScriptConfig"]:
+        """
+        Method that is parsing inline script metadata from datachain script and
+        instantiating ScriptConfig class with found data. If no inline metadata is
+        found, it returns None
+        """
+        try:
+            meta = ScriptConfig.read(script)
+            if not meta:
+                return None
+            custom = meta.get("tools", {}).get("datachain", {})
+            return ScriptConfig(
+                python_version=meta.get("requires-python"),
+                dependencies=meta.get("dependencies"),
+                num_workers=custom.get("workers", {}).get("num_workers"),
+                attachments=custom.get("attachments"),
+                params={k: str(v) for k, v in custom.get("params").items()}
+                if custom.get("params")
+                else None,
+                inputs=custom.get("inputs"),
+                outputs=custom.get("outputs"),
+            )
+        except Exception as e:
+            raise ScriptConfigParsingError(
+                f"Error when parsing script meta: {e}"
+            ) from e

{datachain-0.10.0.dist-info → datachain-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.10.0
+Version: 0.11.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -49,6 +49,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
 Requires-Dist: websockets
+Requires-Dist: tomli; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -102,7 +103,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.74; extra == "examples"
+Requires-Dist: ultralytics==8.3.78; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================

{datachain-0.10.0.dist-info → datachain-0.11.0.dist-info}/RECORD

@@ -12,6 +12,7 @@ datachain/nodes_fetcher.py,sha256=_wgaKyqEjkqdwJ_Hj6D8vUYz7hnU7g6xhm0H6ZnYxmE,10
 datachain/nodes_thread_pool.py,sha256=uPo-xl8zG5m9YgODjPFBpbcqqHjI-dcxH87yAbj_qco,3192
 datachain/progress.py,sha256=lRzxoYP4Qv2XBwD78sOkmYRzHFpZ2ExVNJF8wAeICtY,770
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/script_meta.py,sha256=V-LaFOZG84pD0Zc0NvejYdzwDgzITv6yHvAHggDCnuY,4978
 datachain/studio.py,sha256=Coo_6murSjh-RypiHDWNsVXGmfsopyMPCpPS1sA6uUc,9844
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=n8fcyOM8P_2CEFK4h8BZxCAwCkOpt8NAeJK5tm1gIOg,14433
@@ -69,7 +70,7 @@ datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=zS4lmXHVBXc9ntcyea2a1CRLXGSAN_0glXcF88CohgY,2685
 datachain/lib/dataset_info.py,sha256=IjdF1E0TQNOq9YyynfWiCFTeZpbyGfyJvxgJY4YN810,2493
 datachain/lib/dc.py,sha256=QQPnrS_OB1d3CfjLnYtRByGc7wNX_YT24WOjaoFPJgw,95372
-datachain/lib/file.py,sha256=8OblP_hYJLh0z7MWGo3AiyO48eEJ13tzgla1UQf9A8I,27517
+datachain/lib/file.py,sha256=Bbnb7JBiAFRD1RsZwPdvoiWFKHkl7V3haDLh672xTZg,27658
 datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
 datachain/lib/image.py,sha256=AMXYwQsmarZjRbPCZY3M1jDsM2WAB_b3cTY4uOIuXNU,2675
 datachain/lib/listing.py,sha256=auodM0HitYZsL0DybdgQUYhne_LgkVW-LKGYYOACP90,7272
@@ -135,9 +136,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=z3zRJNzjWrpPuRw-zgFbCOBKInyYxJew8ygrYQRQLNc,2930
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.10.0.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.10.0.dist-info/METADATA,sha256=4Eoe6lnoy_HBYtdzrAIjNnagKXagattQ_mluP9WC-ek,11195
-datachain-0.10.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-datachain-0.10.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.10.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.10.0.dist-info/RECORD,,
+datachain-0.11.0.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.11.0.dist-info/METADATA,sha256=ijLSRDc7IAZe6YxdX0ZRRNY2LOUlsFFib660U_upu20,11241
+datachain-0.11.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+datachain-0.11.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.11.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.11.0.dist-info/RECORD,,

{datachain-0.10.0.dist-info → datachain-0.11.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any