datachain 0.18.8__py3-none-any.whl → 0.18.9__py3-none-any.whl

datachain/cache.py

@@ -76,9 +76,9 @@ class Cache:
     async def download(
         self, file: "File", client: "Client", callback: Optional[Callback] = None
     ) -> None:
+        from_path = f"{file.source}/{file.path}"
         from dvc_objects.fs.utils import tmp_fname
 
-        from_path = file.get_uri()
         odb_fs = self.odb.fs
         tmp_info = odb_fs.join(self.odb.tmp_dir, tmp_fname())  # type: ignore[arg-type]
         size = file.size

datachain/client/fsspec.py

@@ -207,14 +207,13 @@ class Client(ABC):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        file_path = file.get_path_normalized()
         kwargs = {}
         if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
-            self.get_full_path(file_path, file.version), **kwargs
+            self.get_full_path(file.path, file.version), **kwargs
         )
-        return self.info_to_file(info, file_path).etag
+        return self.info_to_file(info, file.path).etag
 
     def get_file_info(self, path: str, version_id: Optional[str] = None) -> "File":
         info = self.fs.info(self.get_full_path(path, version_id), version_id=version_id)
@@ -386,8 +385,7 @@ class Client(ABC):
             return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(
-            self.fs.open(self.get_full_path(file.get_path_normalized(), file.version)),
-            cb,
+            self.fs.open(self.get_full_path(file.path, file.version)), cb
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":

datachain/client/local.py

@@ -99,7 +99,7 @@ class FileClient(Client):
         )
 
     async def get_current_etag(self, file: "File") -> str:
-        info = self.fs.info(self.get_full_path(file.get_path_normalized()))
+        info = self.fs.info(self.get_full_path(file.path))
         return self.info_to_file(info, "").etag
 
     async def get_size(self, path: str, version_id: Optional[str] = None) -> int:
@@ -138,8 +138,8 @@ class FileClient(Client):
         if not self.use_symlinks:
             super().fetch_nodes(nodes, shared_progress_bar)
 
-    def do_instantiate_object(self, file: File, dst: str) -> None:
+    def do_instantiate_object(self, uid, dst):
         if self.use_symlinks:
-            os.symlink(Path(self.name, file.path), dst)
+            os.symlink(Path(self.name, uid.path), dst)
         else:
-            super().do_instantiate_object(file, dst)
+            super().do_instantiate_object(uid, dst)

datachain/lib/arrow.py

@@ -76,7 +76,7 @@ class ArrowGenerator(Generator):
             fs_path = file.path
             fs = ReferenceFileSystem({fs_path: [cache_path]})
         else:
-            fs, fs_path = file.get_fs(), file.get_fs_path()
+            fs, fs_path = file.get_fs(), file.get_path()
 
         kwargs = self.kwargs
         if format := kwargs.get("format"):
@@ -161,7 +161,7 @@ def infer_schema(chain: "DataChain", **kwargs) -> pa.Schema:
 
     schemas = []
     for file in chain.collect("file"):
-        ds = dataset(file.get_fs_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
+        ds = dataset(file.get_path(), filesystem=file.get_fs(), **kwargs)  # type: ignore[union-attr]
         schemas.append(ds.schema)
     if not schemas:
         raise ValueError(

datachain/lib/file.py

@@ -5,14 +5,13 @@ import json
 import logging
 import os
 import posixpath
-import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
 from io import BytesIO
-from pathlib import Path, PurePath, PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
@@ -70,7 +69,7 @@ class FileExporter(NodesThreadPool):
         for task in done:
             task.result()
 
-    def do_task(self, file: "File"):
+    def do_task(self, file):
         file.export(
             self.output,
             self.placement,
@@ -257,8 +256,8 @@ class File(DataModel):
 
     @field_validator("path", mode="before")
     @classmethod
-    def validate_path(cls, path: str) -> str:
-        return PurePath(path).as_posix() if path else ""
+    def validate_path(cls, path):
+        return Path(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()
@@ -320,11 +319,11 @@ class File(DataModel):
         return cls(**{key: row[key] for key in cls._datachain_column_types})
 
     @property
-    def name(self) -> str:
+    def name(self):
         return PurePosixPath(self.path).name
 
     @property
-    def parent(self) -> str:
+    def parent(self):
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
@@ -367,7 +366,7 @@ class File(DataModel):
 
         client.upload(self.read(), destination)
 
-    def _symlink_to(self, destination: str) -> None:
+    def _symlink_to(self, destination: str):
         if self.location:
             raise OSError(errno.ENOTSUP, "Symlinking virtual file is not supported")
 
@@ -376,7 +375,7 @@ class File(DataModel):
             source = self.get_local_path()
             assert source, "File was not cached"
         elif self.source.startswith("file://"):
-            source = self.get_fs_path()
+            source = self.get_path()
         else:
             raise OSError(errno.EXDEV, "can't link across filesystems")
 
@@ -453,62 +452,27 @@ class File(DataModel):
 
     def get_file_ext(self):
         """Returns last part of file name without `.`."""
-        return PurePosixPath(self.path).suffix.lstrip(".")
+        return PurePosixPath(self.path).suffix.strip(".")
 
     def get_file_stem(self):
         """Returns file name without extension."""
         return PurePosixPath(self.path).stem
 
     def get_full_name(self):
-        """
-        [DEPRECATED] Use `file.path` directly instead.
-
-        Returns name with parent directories.
-        """
-        warnings.warn(
-            "file.get_full_name() is deprecated and will be removed "
-            "in a future version. Use `file.path` directly.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
+        """Returns name with parent directories."""
         return self.path
 
-    def get_path_normalized(self) -> str:
-        if not self.path:
-            raise FileError("path must not be empty", self.source, self.path)
-
-        if self.path.endswith("/"):
-            raise FileError("path must not be a directory", self.source, self.path)
-
-        normpath = os.path.normpath(self.path)
-        normpath = PurePath(normpath).as_posix()
-
-        if normpath == ".":
-            raise FileError("path must not be a directory", self.source, self.path)
-
-        if any(part == ".." for part in PurePath(normpath).parts):
-            raise FileError("path must not contain '..'", self.source, self.path)
-
-        return normpath
-
-    def get_uri(self) -> str:
+    def get_uri(self):
         """Returns file URI."""
-        return f"{self.source}/{self.get_path_normalized()}"
-
-    def get_fs_path(self) -> str:
-        """
-        Returns file path with respect to the filescheme.
-
-        If `normalize` is True, the path is normalized to remove any redundant
-        separators and up-level references.
+        return f"{self.source}/{self.get_full_name()}"
 
-        If the file scheme is "file", the path is converted to a local file path
-        using `url2pathname`. Otherwise, the original path with scheme is returned.
-        """
+    def get_path(self) -> str:
+        """Returns file path."""
         path = unquote(self.get_uri())
-        path_parsed = urlparse(path)
-        if path_parsed.scheme == "file":
-            path = url2pathname(path_parsed.path)
+        source = urlparse(self.source)
+        if source.scheme == "file":
+            path = urlparse(path).path
+            path = url2pathname(path)
         return path
 
     def get_destination_path(
@@ -523,7 +487,7 @@ class File(DataModel):
         elif placement == "etag":
             path = f"{self.etag}{self.get_file_suffix()}"
         elif placement == "fullpath":
-            path = unquote(self.get_path_normalized())
+            path = unquote(self.get_full_name())
             source = urlparse(self.source)
             if source.scheme and source.scheme != "file":
                 path = posixpath.join(source.netloc, path)
@@ -561,9 +525,8 @@ class File(DataModel):
             ) from e
 
         try:
-            normalized_path = self.get_path_normalized()
-            info = client.fs.info(client.get_full_path(normalized_path))
-            converted_info = client.info_to_file(info, normalized_path)
+            info = client.fs.info(client.get_full_path(self.path))
+            converted_info = client.info_to_file(info, self.path)
             return type(self)(
                 path=self.path,
                 source=self.source,
@@ -574,17 +537,8 @@ class File(DataModel):
                 last_modified=converted_info.last_modified,
                 location=self.location,
             )
-        except FileError as e:
-            logger.warning(
-                "File error when resolving %s/%s: %s", self.source, self.path, str(e)
-            )
         except (FileNotFoundError, PermissionError, OSError) as e:
-            logger.warning(
-                "File system error when resolving %s/%s: %s",
-                self.source,
-                self.path,
-                str(e),
-            )
+            logger.warning("File system error when resolving %s: %s", self.path, str(e))
 
         return type(self)(
             path=self.path,
@@ -600,8 +554,6 @@ class File(DataModel):
 
 def resolve(file: File) -> File:
     """
-    [DEPRECATED] Use `file.resolve()` directly instead.
-
     Resolve a File object by checking its existence and updating its metadata.
 
     This function is a wrapper around the File.resolve() method, designed to be
@@ -617,12 +569,6 @@ def resolve(file: File) -> File:
         RuntimeError: If the file's catalog is not set or if
         the file source protocol is unsupported.
     """
-    warnings.warn(
-        "resolve() is deprecated and will be removed "
-        "in a future version. Use file.resolve() directly.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
     return file.resolve()
 
 
@@ -970,7 +916,7 @@ class ArrowRow(DataModel):
             ds = dataset(path, **self.kwargs)
 
         else:
-            path = self.file.get_fs_path()
+            path = self.file.get_path()
             ds = dataset(path, filesystem=self.file.get_fs(), **self.kwargs)
 
         return ds.take([self.index]).to_reader()

datachain/lib/tar.py

@@ -6,11 +6,12 @@ from datachain.lib.file import File, TarVFile
 
 
 def build_tar_member(parent: File, info: tarfile.TarInfo) -> File:
+    new_parent = parent.get_full_name()
     etag_string = "-".join([parent.etag, info.name, str(info.mtime)])
     etag = hashlib.md5(etag_string.encode(), usedforsecurity=False).hexdigest()
     return File(
         source=parent.source,
-        path=f"{parent.name}/{info.name}",
+        path=f"{new_parent}/{info.name}",
         version=parent.version,
         size=info.size,
         etag=etag,

datachain/lib/webdataset.py

@@ -35,7 +35,7 @@ warnings.filterwarnings(
 
 class WDSError(DataChainError):
     def __init__(self, tar_stream, message: str):
-        super().__init__(f"WebDataset error '{tar_stream.name}': {message}")
+        super().__init__(f"WebDataset error '{tar_stream.get_full_name()}': {message}")
 
 
 class CoreFileDuplicationError(WDSError):

{datachain-0.18.8.dist-info → datachain-0.18.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.8
+Version: 0.18.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.8.dist-info → datachain-0.18.9.dist-info}/RECORD

@@ -1,7 +1,7 @@
 datachain/__init__.py,sha256=Dx_Dw6AuvC_CZtXxfRv0Z-ND6ieC4Cz-tZkMW-Rvmz4,1496
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=RH_jFwJcTXxhEFomaI9yL6S3Onau6NZ6FSKfKFGtrJE,9689
-datachain/cache.py,sha256=3GWMvF2LMpz2l5lWbtbpmzSB-92eGCCtujeWlFa3r14,3609
+datachain/cache.py,sha256=yQblPhOh_Mq74Ma7xT1CL1idLJ0HgrQxpGVYvRy_9Eg,3623
 datachain/config.py,sha256=g8qbNV0vW2VEKpX-dGZ9pAn0DAz6G2ZFcr7SAV3PoSM,4272
 datachain/dataset.py,sha256=XUZ-kSBL1y6juFqlSWXXbattGS1E53lXpyhc0Ip1_AA,20527
 datachain/delta.py,sha256=q-ritPMxgsTh53qJYd2N1TqZ3Inxc7GJ9JED9rE-Z1M,3994
@@ -39,10 +39,10 @@ datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
 datachain/client/azure.py,sha256=7yyAgANHfu9Kfh187MKNTT1guvu9Q-WYsi4vYoY3aew,3270
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=SSKhvl7x2IzECYUsJ_4hYxvy46AiU0wpsfPduE9alFI,13995
+datachain/client/fsspec.py,sha256=c8oRBUMo31k8bMB_mIA60PDfna4nYTdslzHqmqL2Uvg,13918
 datachain/client/gcs.py,sha256=8hcFhEHp8qGRsJoyfCoawfuwb1Et-MSkyQoM9AnNuXI,5204
 datachain/client/hf.py,sha256=posnI5WOKOMG1yY_ZiV9Orcd24QsUPKZlOXgJVLxxrM,1558
-datachain/client/local.py,sha256=0J52Wzvw25hSucVlzBvLuMRAZwrAHZAYDvD1mNBqf4c,4607
+datachain/client/local.py,sha256=cGoCYflribzexiOe-Y1qbaE2fJRh-_EgQrfCSa0yK_E,4568
 datachain/client/s3.py,sha256=6DNVGLg-woPS1DVlYVX2rIlunNblsuxyOnI1rSzhW3k,7515
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
@@ -68,11 +68,11 @@ datachain/func/random.py,sha256=t7jwXsI8-hy0qAdvjAntgzy-AHtTAfozlZ1CpKR-QZE,458
 datachain/func/string.py,sha256=X9u4ip97U63RCaKRhMddoze7HgPiY3LbPRn9G06UWWo,7311
 datachain/func/window.py,sha256=ImyRpc1QI8QUSPO7KdD60e_DPVo7Ja0G5kcm6BlyMcw,1584
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=K8djofgt4HEgxnkwqZZChccAqeIQ_1D2urGyqti-1-4,10259
+datachain/lib/arrow.py,sha256=mFO_6wRqzpEzBhXf7Xn1aeLUvaiHcC6XQ-8as9sbcgY,10253
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=ZwBXELtqROEdLL4DmxTipnwUZmhQvMz_UVDzyf7nQ9Y,2899
 datachain/lib/dataset_info.py,sha256=d-jz6zeDU5DEgYtyeSF5nK0MU-40FV5km_iOCh4pXzo,3179
-datachain/lib/file.py,sha256=-Y0ccgfQt-2jOnNhOH5j5fTQpCsS9z2ja97umDUHbmA,33054
+datachain/lib/file.py,sha256=mzc7_fpHAkVhs4z3jBUhFQzPEbODdXJpzjVfby2IkC4,31117
 datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
 datachain/lib/image.py,sha256=erWvZW5M3emnbl6_fGAOPyKm-1EKbt3vOdWPfe3Oo7U,3265
 datachain/lib/listing.py,sha256=5_GoATtIwCtd1JMqlorPB_vQDxndOQZpiWjNOG3NMw4,7007
@@ -82,13 +82,13 @@ datachain/lib/model_store.py,sha256=DNIv8Y6Jtk1_idNLzIpsThOsdW2BMAudyUCbPUcgcxk,
 datachain/lib/pytorch.py,sha256=elrmJ4YUDC2LZ9yXM1KwImVBOYIBJf6k0ZR7eSe6Aao,7712
 datachain/lib/settings.py,sha256=ZELRCTLbi5vzRPiDX6cQ9LLg9TefJ_A05gIGni0lll8,2535
 datachain/lib/signal_schema.py,sha256=Zhg8qThFDf9eoNWFH6KGeYB-sIGys7A_ybq2CUBG7Dg,36127
-datachain/lib/tar.py,sha256=k8RFnF72H1jxbMghQQbmoGL-UsA1im8gRLXBM1GJAYI,999
+datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
 datachain/lib/udf.py,sha256=FWqA476ygdk4MU-0qehYKxvnt8Tekh21Cyf3RgddD1k,16674
 datachain/lib/udf_signature.py,sha256=2EtsOPDNSPqcOlYwqbCdy6RF5MldI-7smii8aLy8p7Y,7543
 datachain/lib/utils.py,sha256=rG2y7NwTqZOuomZZRmrA-Q-ANM_j1cToQYqDJoOeGyU,1480
 datachain/lib/video.py,sha256=u6fLJWj5G6QqsVkpfHnKGklBNpG3BRRg6v3izngnNcU,6767
-datachain/lib/webdataset.py,sha256=hZWar13LoZ1TAidFW_sl9rUO-KtMJQY3OFmbnPkJw_A,6913
+datachain/lib/webdataset.py,sha256=o7SHk5HOUWsZ5Ln04xOM04eQqiBHiJNO7xLgyVBrwo8,6924
 datachain/lib/webdataset_laion.py,sha256=xvT6m_r5y0KbOx14BUe7UC5mOgrktJq53Mh-H0EVlUE,2525
 datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/convert/flatten.py,sha256=IZFiUYbgXSxXhPSG5Cqf5IjnJ4ZDZKXMr4o_yCR1NY4,1505
@@ -153,9 +153,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.18.8.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.18.8.dist-info/METADATA,sha256=7_EQNrTrI5u-hjaGNfJOamf3LW-qljTmCuELCFkA2yE,11319
-datachain-0.18.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.18.8.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.18.8.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.18.8.dist-info/RECORD,,
+datachain-0.18.9.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.18.9.dist-info/METADATA,sha256=0BhJEeQiYf41Rg7DLgJ-WtiUu9cpwwUtVwo__lPaMAw,11319
+datachain-0.18.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.18.9.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.18.9.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.18.9.dist-info/RECORD,,