datachain 0.20.1__tar.gz → 0.20.3__tar.gz

{datachain-0.20.1 → datachain-0.20.3}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.11.13'
+    rev: 'v0.12.0'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.20.1 → datachain-0.20.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.20.1
+Version: 0.20.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.20.1 → datachain-0.20.3}/examples/get_started/json-csv-reader.py

@@ -48,7 +48,7 @@ def main():
 
     # Print JSON schema in Pydantic format from main COCO annotation
     chain = dc.read_storage(uri, anon="True").filter(dc.C("file.path").glob("*.json"))
-    file = next(chain.limit(1).collect("file"))
+    file = chain.limit(1).to_values("file")[0]
     print(gen_datamodel_code(file, jmespath="@", model_name="Coco"))
 
     # Static JSON schema test parsing 3/7 objects

{datachain-0.20.1 → datachain-0.20.3}/examples/incremental_processing/delta.py

@@ -47,7 +47,7 @@ def process_files_with_delta():
     print("\nDataset versions:")
     test_dataset = dc.datasets().filter(C("name") == "test_files")
 
-    for version in test_dataset.collect("version"):
+    for version in test_dataset.to_iter("version"):
         print(f"- Version: {version}")
 
     # Show the last 3 records to demonstrate the incremental processing

{datachain-0.20.1 → datachain-0.20.3}/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -1,3 +1,5 @@
+import os
+
 from huggingface_hub import InferenceClient
 from requests import HTTPError
 
@@ -23,6 +25,7 @@ def eval_dialog(
 ) -> DialogEval:
     try:
         completion = client.chat_completion(
+            model="meta-llama/Llama-3.3-70B-Instruct",
             messages=[
                 {
                     "role": "user",
@@ -31,9 +34,10 @@ def eval_dialog(
             ],
             response_format={"type": "json", "value": DialogEval.model_json_schema()},
         )
-    except HTTPError:
+    except HTTPError as e:
         return DialogEval(
-            result="Error", reason="Error while interacting with the Hugging Face API."
+            result="Error",
+            reason=f"Error while interacting with the Hugging Face API. {e}",
         )
 
     message = completion.choices[0].message
@@ -48,9 +52,15 @@ def eval_dialog(
 # Save to HF as Parquet. Dataset can be previewed here:
 # https://huggingface.co/datasets/dvcorg/test-datachain-llm-eval/viewer
 (
-    dc.read_csv("hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv")
-    .settings(parallel=10)
-    .setup(client=lambda: InferenceClient("meta-llama/Llama-3.1-70B-Instruct"))
+    dc.read_csv(
+        "hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv", source=False
+    )
+    .settings(parallel=True)
+    .setup(
+        client=lambda: InferenceClient(
+            provider="hf-inference", api_key=os.environ["HF_TOKEN"]
+        )
+    )
     .map(response=eval_dialog)
     .to_parquet("hf://datasets/dvcorg/test-datachain-llm-eval/data.parquet")
 )

{datachain-0.20.1 → datachain-0.20.3}/pyproject.toml

@@ -221,7 +221,8 @@ ignore = [
   "PERF203",  # perflint - try-except-in-loop, irrelevant for Python>=3.11
   "PERF401",
   "D100",  # undocumented-public-module
-  "D205"  # one-blank-line-after-class
+  "D205",  # one-blank-line-after-class
+  "PLC0415"  # import-outside-top-level
 ]
 select = [
   "B",  # flake8-bugbear

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/__init__.py

@@ -1,4 +1,3 @@
-from datachain.lib import namespaces, projects
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import (
     C,
@@ -33,6 +32,7 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
 from datachain.query import metrics, param
@@ -63,14 +63,13 @@ __all__ = [
     "VideoFile",
     "VideoFragment",
     "VideoFrame",
+    "create_project",
     "datasets",
     "delete_dataset",
     "is_chain_type",
     "listings",
     "metrics",
-    "namespaces",
     "param",
-    "projects",
     "read_csv",
     "read_database",
     "read_dataset",

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/cache.py

@@ -39,7 +39,7 @@ def temporary_cache(
             cache.destroy()
 
 
-class Cache:
+class Cache:  # noqa: PLW1641
     def __init__(self, cache_dir: str, tmp_dir: str):
         self.odb = LocalHashFileDB(
             LocalFileSystem(),
@@ -76,9 +76,9 @@ class Cache:
     async def download(
         self, file: "File", client: "Client", callback: Optional[Callback] = None
     ) -> None:
-        from_path = f"{file.source}/{file.path}"
         from dvc_objects.fs.utils import tmp_fname
 
+        from_path = file.get_uri()
         odb_fs = self.odb.fs
         tmp_info = odb_fs.join(self.odb.tmp_dir, tmp_fname())  # type: ignore[arg-type]
         size = file.size

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/catalog/catalog.py

@@ -1491,13 +1491,13 @@ class Catalog:
 
         namespace = self.metastore.create_namespace(
             remote_ds.project.namespace.name,
-            description=remote_ds.project.namespace.description,
+            description=remote_ds.project.namespace.descr,
             uuid=remote_ds.project.namespace.uuid,
         )
         project = self.metastore.create_project(
-            remote_ds.project.name,
             namespace.name,
-            description=remote_ds.project.description,
+            remote_ds.project.name,
+            description=remote_ds.project.descr,
             uuid=remote_ds.project.uuid,
         )
 

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/cli/commands/ls.py

@@ -63,8 +63,8 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = list(listings().collect("listing"))
-        for ls in listing:
+        listing = listings().to_list("listing")
+        for (ls,) in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 
 

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/client/fsspec.py

@@ -207,13 +207,14 @@ class Client(ABC):
         )
 
     async def get_current_etag(self, file: "File") -> str:
+        file_path = file.get_path_normalized()
         kwargs = {}
         if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
-            self.get_full_path(file.path, file.version), **kwargs
+            self.get_full_path(file_path, file.version), **kwargs
         )
-        return self.info_to_file(info, file.path).etag
+        return self.info_to_file(info, file_path).etag
 
     def get_file_info(self, path: str, version_id: Optional[str] = None) -> "File":
         info = self.fs.info(self.get_full_path(path, version_id), version_id=version_id)
@@ -386,7 +387,8 @@ class Client(ABC):
             return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(
-            self.fs.open(self.get_full_path(file.path, file.version)), cb
+            self.fs.open(self.get_full_path(file.get_path_normalized(), file.version)),
+            cb,
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/client/hf.py

@@ -21,6 +21,9 @@ def _wrap_class(sync_fs_class):
     asynchronous to False by default. This is similar to other Async FS
     we initialize. E.g. it means we don't break things in Jupyter where code
     run in async.
+
+    This also fixes write operations by ensuring they are properly forwarded
+    to the underlying filesystem without async buffering issues.
     """
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -29,6 +32,13 @@ def _wrap_class(sync_fs_class):
             sync_fs = sync_fs_class(*args, **kwargs)
             super().__init__(sync_fs, asynchronous=False)
 
+        def open(self, path, mode="rb", **kwargs):
+            # Override open to ensure write operations work correctly.
+            # It seems to be a bug in the fsspec wrapper. It avoids
+            # wrapping open() explicitly but also doesn't redirect it to
+            # sync filesystem.
+            return self.sync_fs.open(path, mode, **kwargs)
+
     GeneratedAsyncFileSystemWrapper.__name__ = f"Async{sync_fs_class.__name__}Wrapper"
     return GeneratedAsyncFileSystemWrapper
 

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/client/local.py

@@ -99,7 +99,7 @@ class FileClient(Client):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        info = self.fs.info(self.get_full_path(file.path))
+        info = self.fs.info(self.get_full_path(file.get_path_normalized()))
         return self.info_to_file(info, "").etag
 
     async def get_size(self, path: str, version_id: Optional[str] = None) -> int:
@@ -138,8 +138,8 @@ class FileClient(Client):
         if not self.use_symlinks:
             super().fetch_nodes(nodes, shared_progress_bar)
 
-    def do_instantiate_object(self, uid, dst):
+    def do_instantiate_object(self, file: File, dst: str) -> None:
         if self.use_symlinks:
-            os.symlink(Path(self.name, uid.path), dst)
+            os.symlink(Path(self.name, file.path), dst)
         else:
-            super().do_instantiate_object(uid, dst)
+            super().do_instantiate_object(file, dst)

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/data_storage/metastore.py

@@ -185,8 +185,8 @@ class AbstractMetastore(ABC, Serializable):
     @abstractmethod
     def create_project(
         self,
-        name: str,
         namespace_name: str,
+        name: str,
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
@@ -195,8 +195,13 @@ class AbstractMetastore(ABC, Serializable):
         """Creates new project in specific namespace"""
 
     @abstractmethod
-    def get_project(self, name: str, namespace_name: str, conn=None) -> Project:
-        """Gets a single project inside some namespace by name"""
+    def get_project(
+        self, name: str, namespace_name: str, create: bool = False, conn=None
+    ) -> Project:
+        """
+        Gets a single project inside some namespace by name.
+        It also creates project if not found and create flag is set to True.
+        """
 
     @abstractmethod
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
@@ -763,14 +768,18 @@ class AbstractDBMetastore(AbstractMetastore):
 
     def create_project(
         self,
-        name: str,
         namespace_name: str,
+        name: str,
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
         **kwargs,
     ) -> Project:
-        namespace = self.get_namespace(namespace_name)
+        try:
+            namespace = self.get_namespace(namespace_name)
+        except NamespaceNotFoundError:
+            namespace = self.create_namespace(namespace_name)
+
         query = self._projects_insert().values(
             namespace_id=namespace.id,
             uuid=uuid or str(uuid4()),
@@ -788,7 +797,9 @@ class AbstractDBMetastore(AbstractMetastore):
 
         return self.get_project(name, namespace.name)
 
-    def get_project(self, name: str, namespace_name: str, conn=None) -> Project:
+    def get_project(
+        self, name: str, namespace_name: str, create: bool = False, conn=None
+    ) -> Project:
         """Gets a single project inside some namespace by name"""
         n = self._namespaces
         p = self._projects
@@ -803,6 +814,8 @@ class AbstractDBMetastore(AbstractMetastore):
 
         rows = list(self.db.execute(query, conn=conn))
         if not rows:
+            if create:
+                return self.create_project(namespace_name, name)
             raise ProjectNotFoundError(
                 f"Project {name} in namespace {namespace_name} not found."
             )

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/data_storage/sqlite.py

@@ -469,10 +469,10 @@ class SQLiteMetastore(AbstractDBMetastore):
         Studio dataset.
         """
         system_namespace = self.create_namespace(Namespace.system(), "System namespace")
-        self.create_project(Project.listing(), system_namespace.name, "Listing project")
+        self.create_project(system_namespace.name, Project.listing(), "Listing project")
 
         local_namespace = self.create_namespace(Namespace.default(), "Local namespace")
-        self.create_project(Project.default(), local_namespace.name, "Local project")
+        self.create_project(local_namespace.name, Project.default(), "Local project")
 
     def _check_schema_version(self) -> None:
         """

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/dataset.py

@@ -83,10 +83,11 @@ def parse_dataset_name(name: str) -> tuple[Optional[str], Optional[str], str]:
     if not name:
         raise ValueError("Name must be defined to parse it")
     split = name.split(".")
-    if len(split) == 3:
-        return tuple(split)  # type: ignore[return-value]
+    name = split[-1]
+    project_name = split[-2] if len(split) > 1 else None
+    namespace_name = split[-3] if len(split) > 2 else None
 
-    return None, None, name
+    return namespace_name, project_name, name
 
 
 class DatasetDependencyType:

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/delta.py

@@ -62,7 +62,7 @@ def _get_delta_chain(
     )
 
     # Calculate diff between source versions
-    return source_dc_latest.compare(source_dc, on=on, compare=compare, deleted=False)
+    return source_dc_latest.diff(source_dc, on=on, compare=compare, deleted=False)
 
 
 def _get_retry_chain(
@@ -237,7 +237,7 @@ def delta_retry_update(
         return None, None, False
 
     latest_dataset = datachain.read_dataset(name, version=latest_version)
-    compared_chain = latest_dataset.compare(
+    compared_chain = latest_dataset.diff(
         processing_chain,
         on=right_on or on,
         added=True,

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/func/func.py

@@ -25,7 +25,7 @@ if TYPE_CHECKING:
 ColT = Union[str, Column, ColumnElement, "Func", tuple]
 
 
-class Func(Function):
+class Func(Function):  # noqa: PLW1641
     """Represents a function to be applied to a column in a SQL query."""
 
     def __init__(

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/lib/arrow.py

@@ -76,7 +76,7 @@ class ArrowGenerator(Generator):
             fs_path = file.path
             fs = ReferenceFileSystem({fs_path: [cache_path]})
         else:
-            fs, fs_path = file.get_fs(), file.get_path()
+            fs, fs_path = file.get_fs(), file.get_fs_path()
 
         kwargs = self.kwargs
         if format := kwargs.get("format"):
@@ -160,8 +160,8 @@ def infer_schema(chain: "DataChain", **kwargs) -> pa.Schema:
         kwargs["format"] = fix_pyarrow_format(format, parse_options)
 
     schemas = []
-    for file in chain.collect("file"):
-        ds = dataset(file.get_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
+    for (file,) in chain.to_iter("file"):
+        ds = dataset(file.get_fs_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
         schemas.append(ds.schema)
     if not schemas:
         raise ValueError(

{datachain-0.20.1 → datachain-0.20.3}/src/datachain/lib/dataset_info.py

@@ -22,8 +22,8 @@ if TYPE_CHECKING:
 
 class DatasetInfo(DataModel):
     name: str
-    namespace_name: str
-    project_name: str
+    namespace: str
+    project: str
     uuid: str = Field(default=str(uuid4()))
     version: str = Field(default=DEFAULT_DATASET_VERSION)
     status: int = Field(default=DatasetStatus.CREATED)
@@ -93,8 +93,8 @@ class DatasetInfo(DataModel):
         return cls(
             uuid=version.uuid,
             name=dataset.name,
-            namespace_name=dataset.project.namespace.name,
-            project_name=dataset.project.name,
+            namespace=dataset.project.namespace.name,
+            project=dataset.project.name,
             version=version.version,
             status=version.status,
             created_at=version.created_at,