datachain 0.3.16__py3-none-any.whl → 0.3.18__py3-none-any.whl

datachain/lib/dc.py

@@ -58,6 +58,7 @@ from datachain.query.dataset import (
 )
 from datachain.query.schema import DEFAULT_DELIMITER, Column, DatasetRow
 from datachain.sql.functions import path as pathfunc
+from datachain.telemetry import telemetry
 from datachain.utils import inside_notebook
 
 if TYPE_CHECKING:
@@ -246,6 +247,9 @@ class DataChain(DatasetQuery):
             **kwargs,
             indexing_column_types=File._datachain_column_types,
         )
+
+        telemetry.send_event_once("class", "datachain_init", **kwargs)
+
         if settings:
             self._settings = Settings(**settings)
         else:
@@ -1337,8 +1341,7 @@ class DataChain(DatasetQuery):
                     other.signals_schema.resolve(*right_on).db_signals(),
                 )  # type: ignore[arg-type]
             )
-
-        return super()._subtract(other, signals)  # type: ignore[arg-type]
+        return super().subtract(other, signals)  # type: ignore[arg-type]
 
     @classmethod
     def from_values(

datachain/lib/file.py

@@ -1,11 +1,14 @@
+import hashlib
 import io
 import json
 import logging
 import os
 import posixpath
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
+from functools import partial
 from io import BytesIO
 from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
@@ -19,7 +22,6 @@ from pydantic import Field, field_validator
 if TYPE_CHECKING:
     from typing_extensions import Self
 
-from datachain.cache import UniqueId
 from datachain.client.fileslice import FileSlice
 from datachain.lib.data_model import DataModel
 from datachain.lib.utils import DataChainError
@@ -27,7 +29,13 @@ from datachain.sql.types import JSON, Boolean, DateTime, Int, String
 from datachain.utils import TIME_ZERO
 
 if TYPE_CHECKING:
+    from typing_extensions import Self
+
     from datachain.catalog import Catalog
+    from datachain.client.fsspec import Client
+    from datachain.dataset import RowDict
+
+sha256 = partial(hashlib.sha256, usedforsecurity=False)
 
 logger = logging.getLogger("datachain")
 
@@ -38,7 +46,7 @@ ExportPlacement = Literal["filename", "etag", "fullpath", "checksum"]
 class VFileError(DataChainError):
     def __init__(self, file: "File", message: str, vtype: str = ""):
         type_ = f" of vtype '{vtype}'" if vtype else ""
-        super().__init__(f"Error in v-file '{file.get_uid().path}'{type_}: {message}")
+        super().__init__(f"Error in v-file '{file.path}'{type_}: {message}")
 
 
 class FileError(DataChainError):
@@ -85,9 +93,8 @@ class TarVFile(VFile):
         tar_file = File(**parent)
         tar_file._set_stream(file._catalog)
 
-        tar_file_uid = tar_file.get_uid()
-        client = file._catalog.get_client(tar_file_uid.storage)
-        fd = client.open_object(tar_file_uid, use_cache=file._caching_enabled)
+        client = file._catalog.get_client(tar_file.source)
+        fd = client.open_object(tar_file, use_cache=file._caching_enabled)
         return FileSlice(fd, offset, size, file.name)
 
 
@@ -181,7 +188,11 @@ class File(DataModel):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._catalog = None
-        self._caching_enabled = False
+        self._caching_enabled: bool = False
+
+    @classmethod
+    def _from_row(cls, row: "RowDict") -> "Self":
+        return cls(**{key: row[key] for key in cls._datachain_column_types})
 
     @property
     def name(self):
@@ -192,19 +203,18 @@ class File(DataModel):
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
-    def open(self, mode: Literal["rb", "r"] = "rb"):
+    def open(self, mode: Literal["rb", "r"] = "rb") -> Iterator[Any]:
         """Open the file and return a file object."""
         if self.location:
             with VFileRegistry.resolve(self, self.location) as f:  # type: ignore[arg-type]
                 yield f
 
         else:
-            uid = self.get_uid()
-            client = self._catalog.get_client(self.source)
             if self._caching_enabled:
-                client.download(uid, callback=self._download_cb)
+                self.ensure_cached()
+            client: Client = self._catalog.get_client(self.source)
             with client.open_object(
-                uid, use_cache=self._caching_enabled, cb=self._download_cb
+                self, use_cache=self._caching_enabled, cb=self._download_cb
             ) as f:
                 yield io.TextIOWrapper(f) if mode == "r" else f
 
@@ -252,23 +262,25 @@ class File(DataModel):
         self._caching_enabled = caching_enabled
         self._download_cb = download_cb
 
-    def get_uid(self) -> UniqueId:
-        """Returns unique ID for file."""
-        dump = self.model_dump()
-        return UniqueId(*(dump[k] for k in self._unique_id_keys))
+    def ensure_cached(self) -> None:
+        if self._catalog is None:
+            raise RuntimeError(
+                "cannot download file to cache because catalog is not setup"
+            )
+        client = self._catalog.get_client(self.source)
+        client.download(self, callback=self._download_cb)
+
+    def get_local_path(self) -> Optional[str]:
+        """Return path to a file in a local cache.
 
-    def get_local_path(self, download: bool = False) -> Optional[str]:
-        """Returns path to a file in a local cache.
-        Return None if file is not cached. Throws an exception if cache is not setup."""
+        Returns None if file is not cached.
+        Raises an exception if cache is not setup.
+        """
         if self._catalog is None:
             raise RuntimeError(
                 "cannot resolve local file path because catalog is not setup"
             )
-        uid = self.get_uid()
-        if download:
-            client = self._catalog.get_client(self.source)
-            client.download(uid, callback=self._download_cb)
-        return self._catalog.cache.get_path(uid)
+        return self._catalog.cache.get_path(self)
 
     def get_file_suffix(self):
         """Returns last part of file name with `.`."""
@@ -323,6 +335,12 @@ class File(DataModel):
         """Returns `fsspec` filesystem for the file."""
         return self._catalog.get_client(self.source).fs
 
+    def get_hash(self) -> str:
+        fingerprint = f"{self.source}/{self.path}/{self.version}/{self.etag}"
+        if self.location:
+            fingerprint += f"/{self.location}"
+        return sha256(fingerprint.encode()).hexdigest()
+
     def resolve(self) -> "Self":
         """
         Resolve a File object by checking its existence and updating its metadata.

datachain/lib/listing.py

@@ -11,6 +11,7 @@ from datachain.client import Client
 from datachain.lib.file import File
 from datachain.query.schema import Column
 from datachain.sql.functions import path as pathfunc
+from datachain.telemetry import telemetry
 from datachain.utils import uses_glob
 
 if TYPE_CHECKING:
@@ -77,8 +78,10 @@ def parse_listing_uri(uri: str, cache, client_config) -> tuple[str, str, str]:
     """
     Parsing uri and returns listing dataset name, listing uri and listing path
     """
+    client_config = client_config or {}
     client = Client.get_client(uri, cache, **client_config)
     storage_uri, path = Client.parse_url(uri)
+    telemetry.log_param("client", client.PREFIX)
 
     # clean path without globs
     lst_uri_path = (

datachain/lib/tar.py

@@ -30,4 +30,5 @@ def process_tar(file: File) -> Iterator[File]:
     with file.open() as fd:
         with tarfile.open(fileobj=fd) as tar:
             for entry in tar.getmembers():
-                yield build_tar_member(file, entry)
+                if entry.isfile():
+                    yield build_tar_member(file, entry)

datachain/listing.py

@@ -156,12 +156,12 @@ class Listing:
 
     def instantiate_nodes(
         self,
-        all_nodes,
+        all_nodes: Iterable[NodeWithPath],
         output,
         total_files=None,
         force=False,
         shared_progress_bar=None,
-    ):
+    ) -> None:
         progress_bar = shared_progress_bar or tqdm(
             desc=f"Instantiating '{output}'",
             unit=" files",
@@ -175,8 +175,8 @@ class Listing:
             dst = os.path.join(output, *node.path)
             dst_dir = os.path.dirname(dst)
             os.makedirs(dst_dir, exist_ok=True)
-            uid = node.n.as_uid(self.client.uri)
-            self.client.instantiate_object(uid, dst, progress_bar, force)
+            file = node.n.to_file(self.client.uri)
+            self.client.instantiate_object(file, dst, progress_bar, force)
             counter += 1
             if counter > 1000:
                 progress_bar.update(counter)

datachain/node.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Optional
 
 import attrs
 
-from datachain.cache import UniqueId
+from datachain.lib.file import File
 from datachain.storage import StorageURI
 from datachain.utils import TIME_ZERO, time_to_str
 
@@ -99,11 +99,11 @@ class Node:
             return self.path + "/"
         return self.path
 
-    def as_uid(self, storage: Optional[StorageURI] = None) -> UniqueId:
-        if storage is None:
-            storage = self.source
-        return UniqueId(
-            storage=storage,
+    def to_file(self, source: Optional[StorageURI] = None) -> File:
+        if source is None:
+            source = self.source
+        return File(
+            source=source,
             path=self.path,
             size=self.size,
             version=self.version or "",
@@ -114,9 +114,23 @@ class Node:
         )
 
     @classmethod
-    def from_dict(cls, d: dict[str, Any]) -> "Self":
-        kw = {f.name: d[f.name] for f in attrs.fields(cls) if f.name in d}
-        return cls(**kw)
+    def from_dict(cls, d: dict[str, Any], file_prefix: str = "file") -> "Self":
+        def _dval(field_name: str):
+            return d.get(f"{file_prefix}__{field_name}")
+
+        return cls(
+            sys__id=d["sys__id"],
+            sys__rand=d["sys__rand"],
+            source=_dval("source"),
+            path=_dval("path"),
+            etag=_dval("etag"),
+            is_latest=_dval("is_latest"),
+            size=_dval("size"),
+            last_modified=_dval("last_modified"),
+            version=_dval("version"),
+            location=_dval("location"),
+            dir_type=DirType.FILE,
+        )
 
     @classmethod
     def from_dir(cls, path, **kwargs) -> "Node":

datachain/nodes_fetcher.py

@@ -1,12 +1,19 @@
 import logging
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
+from datachain.node import Node
 from datachain.nodes_thread_pool import NodesThreadPool
 
+if TYPE_CHECKING:
+    from datachain.cache import DataChainCache
+    from datachain.client.fsspec import Client
+
 logger = logging.getLogger("datachain")
 
 
 class NodesFetcher(NodesThreadPool):
-    def __init__(self, client, max_threads, cache):
+    def __init__(self, client: "Client", max_threads: int, cache: "DataChainCache"):
         super().__init__(max_threads)
         self.client = client
         self.cache = cache
@@ -15,7 +22,7 @@ class NodesFetcher(NodesThreadPool):
         for task in done:
             task.result()
 
-    def do_task(self, chunk):
+    def do_task(self, chunk: Iterable[Node]) -> None:
         from fsspec import Callback
 
         class _CB(Callback):
@@ -23,8 +30,8 @@ class NodesFetcher(NodesThreadPool):
                 self.increase_counter(inc)
 
         for node in chunk:
-            uid = node.as_uid(self.client.uri)
-            if self.cache.contains(uid):
+            file = node.to_file(self.client.uri)
+            if self.cache.contains(file):
                 self.increase_counter(node.size)
             else:
-                self.client.put_in_cache(uid, callback=_CB())
+                self.client.put_in_cache(file, callback=_CB())

datachain/nodes_thread_pool.py

@@ -20,7 +20,7 @@ class NodeChunk:
     def next_downloadable(self):
         node = next(self.nodes, None)
         while node and (
-            not node.is_downloadable or self.cache.contains(node.as_uid(self.storage))
+            not node.is_downloadable or self.cache.contains(node.to_file(self.storage))
         ):
             node = next(self.nodes, None)
         return node

datachain/progress.py

@@ -1,8 +1,6 @@
 """Manages progress bars."""
 
 import logging
-import os
-import re
 import sys
 from threading import RLock
 from typing import Any, ClassVar
@@ -10,20 +8,12 @@ from typing import Any, ClassVar
 from fsspec.callbacks import TqdmCallback
 from tqdm import tqdm
 
+from datachain.utils import env2bool
+
 logger = logging.getLogger(__name__)
 tqdm.set_lock(RLock())
 
 
-def env2bool(var, undefined=False):
-    """
-    undefined: return value if env var is unset
-    """
-    var = os.getenv(var, None)
-    if var is None:
-        return undefined
-    return bool(re.search("1|y|yes|true", var, flags=re.IGNORECASE))
-
-
 class Tqdm(tqdm):
     """
     maximum-compatibility tqdm-based progressbars

datachain/query/__init__.py

@@ -2,7 +2,6 @@ from .dataset import DatasetQuery
 from .params import param
 from .schema import C, DatasetRow, LocalFilename, Object, Stream
 from .session import Session
-from .udf import udf
 
 __all__ = [
     "C",
@@ -13,5 +12,4 @@ __all__ = [
     "Session",
     "Stream",
     "param",
-    "udf",
 ]