datachain 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

datachain/client/fsspec.py

@@ -390,7 +390,7 @@ class Client(ABC):
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":
-        full_path = self.get_full_path(path)
+        full_path = path if path.startswith(self.PREFIX) else self.get_full_path(path)
 
         parent = posixpath.dirname(full_path)
         self.fs.makedirs(parent, exist_ok=True)

datachain/lib/dc.py

@@ -411,6 +411,7 @@ class DataChain:
         object_name: str = "file",
         update: bool = False,
         anon: bool = False,
+        client_config: Optional[dict] = None,
     ) -> "Self":
         """Get data from a storage as a list of file with all file attributes.
         It returns the chain itself as usual.
@@ -423,15 +424,32 @@ class DataChain:
             object_name : Created object column name.
             update : force storage reindexing. Default is False.
             anon : If True, we will treat cloud bucket as public one
+            client_config : Optional client configuration for the storage client.
 
         Example:
+            Simple call from s3
             ```py
             chain = DataChain.from_storage("s3://my-bucket/my-dir")
             ```
+
+            With AWS S3-compatible storage
+            ```py
+            chain = DataChain.from_storage(
+                "s3://my-bucket/my-dir",
+                client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
+            )
+            ```
+
+            Pass existing session
+            ```py
+            session = Session.get()
+            chain = DataChain.from_storage("s3://my-bucket/my-dir", session=session)
+            ```
         """
         file_type = get_file_type(type)
 
-        client_config = {"anon": True} if anon else None
+        if anon:
+            client_config = (client_config or {}) | {"anon": True}
         session = Session.get(session, client_config=client_config, in_memory=in_memory)
         cache = session.catalog.cache
         client_config = session.catalog.client_config
@@ -481,25 +499,56 @@ class DataChain:
         version: Optional[int] = None,
         session: Optional[Session] = None,
         settings: Optional[dict] = None,
-        fallback_to_remote: bool = True,
+        fallback_to_studio: bool = True,
     ) -> "Self":
         """Get data from a saved Dataset. It returns the chain itself.
+        If dataset or version is not found locally, it will try to pull it from Studio.
 
         Parameters:
             name : dataset name
             version : dataset version
+            session : Session to use for the chain.
+            settings : Settings to use for the chain.
+            fallback_to_studio : Try to pull dataset from Studio if not found locally.
+                Default is True.
 
         Example:
             ```py
             chain = DataChain.from_dataset("my_cats")
             ```
+
+            ```py
+            chain = DataChain.from_dataset("my_cats", fallback_to_studio=False)
+            ```
+
+            ```py
+            chain = DataChain.from_dataset("my_cats", version=1)
+            ```
+
+            ```py
+            session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
+            settings = {
+                "cache": True,
+                "parallel": 4,
+                "workers": 4,
+                "min_task_size": 1000,
+                "prefetch": 10,
+            }
+            chain = DataChain.from_dataset(
+                name="my_cats",
+                version=1,
+                session=session,
+                settings=settings,
+                fallback_to_studio=True,
+            )
+            ```
         """
         query = DatasetQuery(
             name=name,
             version=version,
             session=session,
             indexing_column_types=File._datachain_column_types,
-            fallback_to_remote=fallback_to_remote,
+            fallback_to_studio=fallback_to_studio,
         )
         telemetry.send_event_once("class", "datachain_init", name=name, version=version)
         if settings:
@@ -2444,7 +2493,7 @@ class DataChain:
         self._setup = self._setup | kwargs
         return self
 
-    def export_files(
+    def to_storage(
         self,
         output: str,
         signal: str = "file",
@@ -2462,6 +2511,13 @@ class DataChain:
             use_cache: If `True`, cache the files before exporting.
             link_type: Method to use for exporting files.
                 Falls back to `'copy'` if symlinking fails.
+
+        Example:
+            Cross cloud transfer
+            ```py
+            ds = DataChain.from_storage("s3://mybucket")
+            ds.to_storage("gs://mybucket", placement="filename")
+            ```
         """
         if placement == "filename" and (
             self._query.distinct(pathfunc.name(C(f"{signal}__path"))).count()

datachain/lib/file.py

@@ -17,6 +17,7 @@ from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
 
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
+from fsspec.utils import stringify_path
 from PIL import Image as PilImage
 from pydantic import Field, field_validator
 
@@ -214,10 +215,13 @@ class File(DataModel):
 
             catalog = get_catalog()
 
-        parent, name = posixpath.split(path)
+        from datachain.client.fsspec import Client
 
-        client = catalog.get_client(parent)
-        file = client.upload(data, name)
+        client_cls = Client.get_implementation(path)
+        source, rel_path = client_cls.split_url(path)
+
+        client = catalog.get_client(client_cls.get_uri(source))
+        file = client.upload(data, rel_path)
         if not isinstance(file, cls):
             file = cls(**file.model_dump())
         file._set_stream(catalog)
@@ -267,8 +271,9 @@ class File(DataModel):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        with open(destination, mode="wb") as f:
-            f.write(self.read())
+        destination = stringify_path(destination)
+        client: Client = self._catalog.get_client(str(destination))
+        client.upload(self.read(), str(destination))
 
     def _symlink_to(self, destination: str):
         if self.location:
@@ -282,6 +287,7 @@ class File(DataModel):
             source = self.get_path()
         else:
             raise OSError(errno.EXDEV, "can't link across filesystems")
+
         return os.symlink(source, destination)
 
     def export(
@@ -296,7 +302,8 @@ class File(DataModel):
             self._caching_enabled = use_cache
         dst = self.get_destination_path(output, placement)
         dst_dir = os.path.dirname(dst)
-        os.makedirs(dst_dir, exist_ok=True)
+        client: Client = self._catalog.get_client(dst_dir)
+        client.fs.makedirs(dst_dir, exist_ok=True)
 
         if link_type == "symlink":
             try:
@@ -493,7 +500,10 @@ class TextFile(File):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        with open(destination, mode="w") as f:
+        destination = stringify_path(destination)
+
+        client: Client = self._catalog.get_client(destination)
+        with client.fs.open(destination, mode="w") as f:
             f.write(self.read_text())
 
 
@@ -507,7 +517,11 @@ class ImageFile(File):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        self.read().save(destination)
+        destination = stringify_path(destination)
+
+        client: Client = self._catalog.get_client(destination)
+        with client.fs.open(destination, mode="wb") as f:
+            self.read().save(f)
 
 
 class Image(DataModel):

datachain/query/dataset.py

@@ -1085,7 +1085,7 @@ class DatasetQuery:
         session: Optional[Session] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
         in_memory: bool = False,
-        fallback_to_remote: bool = True,
+        fallback_to_studio: bool = True,
     ) -> None:
         self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
@@ -1103,7 +1103,7 @@ class DatasetQuery:
 
         self.name = name
 
-        if fallback_to_remote and is_token_set():
+        if fallback_to_studio and is_token_set():
             ds = self.catalog.get_dataset_with_remote_fallback(name, version)
         else:
             ds = self.catalog.get_dataset(name)

datachain/query/session.py

@@ -139,21 +139,33 @@ class Session:
 
         # Access the active (most recent) context from the stack
         if cls.SESSION_CONTEXTS:
-            return cls.SESSION_CONTEXTS[-1]
+            session = cls.SESSION_CONTEXTS[-1]
 
-        if cls.GLOBAL_SESSION_CTX is None:
+        elif cls.GLOBAL_SESSION_CTX is None:
             cls.GLOBAL_SESSION_CTX = Session(
                 cls.GLOBAL_SESSION_NAME,
                 catalog,
                 client_config=client_config,
                 in_memory=in_memory,
             )
+            session = cls.GLOBAL_SESSION_CTX
 
             atexit.register(cls._global_cleanup)
             cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
             sys.excepthook = cls.except_hook
+        else:
+            session = cls.GLOBAL_SESSION_CTX
 
-        return cls.GLOBAL_SESSION_CTX
+        if client_config and session.catalog.client_config != client_config:
+            session = Session(
+                "session" + uuid4().hex[:4],
+                catalog,
+                client_config=client_config,
+                in_memory=in_memory,
+            )
+            session.__enter__()
+
+        return session
 
     @staticmethod
     def except_hook(exc_type, exc_value, exc_traceback):

{datachain-0.9.0.dist-info → datachain-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.9.0
+Version: 0.10.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -175,7 +175,7 @@ high confidence scores.
 
     likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
                                    & (Column("meta.inference.class_") == "cat"))
-    likely_cats.export_files("high-confidence-cats/", signal="file")
+    likely_cats.to_storage("high-confidence-cats/", signal="file")
 
 
 Example: LLM based text-file evaluation
@@ -216,7 +216,7 @@ Python code:
     )
 
     successful_chain = chain.filter(Column("is_success") == True)
-    successful_chain.export_files("./output_mistral")
+    successful_chain.to_storage("./output_mistral")
 
     print(f"{successful_chain.count()} files were exported")
 

{datachain-0.9.0.dist-info → datachain-0.10.0.dist-info}/RECORD

@@ -36,7 +36,7 @@ datachain/cli/parser/utils.py,sha256=GEzxfPJ4i6nt6JhjvZ3PQesXl9islEV3E-N1NZGrLaA
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
 datachain/client/azure.py,sha256=ma6fJcnveG8wpNy1PSrN5hgvmRdCj8Sf3RKjfd3qCyM,3221
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=whQWKD0tGQUjc8FpA0irxc31wiJuQS4x2arHc98-Lv0,13966
+datachain/client/fsspec.py,sha256=N_n3_DtZuKsLst8-XVda2xYCUHreUU3ld0MNTl8L9f4,14008
 datachain/client/gcs.py,sha256=TY5K5INORKknTnoWDYv0EUztVLmuY1hHmdf2wUB_9uE,5114
 datachain/client/hf.py,sha256=XeVJVbiNViZCpn3sfb90Fr8SYO3BdLmfE3hOWMoqInE,951
 datachain/client/local.py,sha256=Pv67SYdkNkkNExBoKJF9AnNu0FSrt4JqLRkSVsUnveU,4672
@@ -68,8 +68,8 @@ datachain/lib/arrow.py,sha256=9UBCF-lftQaz0yxdsjbLKbyzVSmrF_QSWdhp2oBDPqs,9486
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=zS4lmXHVBXc9ntcyea2a1CRLXGSAN_0glXcF88CohgY,2685
 datachain/lib/dataset_info.py,sha256=IjdF1E0TQNOq9YyynfWiCFTeZpbyGfyJvxgJY4YN810,2493
-datachain/lib/dc.py,sha256=rQZgFLFSIde6wfiAnBnlSE4qnjNjNXQ0F3TGhDQ6ap8,93459
-datachain/lib/file.py,sha256=HXH4pgPN_Zx9rPI0jy-Cjl2F3ItJchH7ycrIXnOgaGk,26892
+datachain/lib/dc.py,sha256=QQPnrS_OB1d3CfjLnYtRByGc7wNX_YT24WOjaoFPJgw,95372
+datachain/lib/file.py,sha256=8OblP_hYJLh0z7MWGo3AiyO48eEJ13tzgla1UQf9A8I,27517
 datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
 datachain/lib/image.py,sha256=AMXYwQsmarZjRbPCZY3M1jDsM2WAB_b3cTY4uOIuXNU,2675
 datachain/lib/listing.py,sha256=auodM0HitYZsL0DybdgQUYhne_LgkVW-LKGYYOACP90,7272
@@ -103,13 +103,13 @@ datachain/model/ultralytics/pose.py,sha256=71KBTcoST2wcEtsyGXqLVpvUtqbp9gwZGA15p
 datachain/model/ultralytics/segment.py,sha256=Z1ab0tZRJubSYNH4KkFlzhYeGNTfAyC71KmkQcToHDQ,2760
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=6w8gzLTmLeylststu-gT5jIqEfi4-djS7_yTYyeo-fw,4190
-datachain/query/dataset.py,sha256=tXinFa-0ytC4j3W3XKjSkVW3qX3iFGQyRO800k9JW98,57083
+datachain/query/dataset.py,sha256=wK_etZkH558pzLKAMBArlj1TQD9n96YK-kpVYBCSR38,57083
 datachain/query/dispatch.py,sha256=_1vjeQ1wjUoxlik55k0JkWqQCUfMjgVWmEOyWRkx0dU,12437
 datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
 datachain/query/schema.py,sha256=b_KnVy6B26Ol4nYG0LqNNpeQ1QYPk95YRGUjXfdaQWs,6606
-datachain/query/session.py,sha256=fQAtl5zRESRDfRS2d5J9KgrWauunCtrd96vP4Ns1KlE,5998
+datachain/query/session.py,sha256=I1KG8jDIaxGAfRfDRucMx8DqsANf_VYWtwtXjeD19lI,6399
 datachain/query/udf.py,sha256=GY8E9pnzPE7ZKl_jvetZpn9R2rlUtMlhoYj4UmrzFzw,594
 datachain/query/utils.py,sha256=u0A_BwG9PNs0DxoDcvSWgWLpj3ByTUv8CqH13CIuGag,1293
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -135,9 +135,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=z3zRJNzjWrpPuRw-zgFbCOBKInyYxJew8ygrYQRQLNc,2930
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.9.0.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.9.0.dist-info/METADATA,sha256=GJFjVJKy6bvK7OwltIhcQEA3GHdpR2pHHJEAdoeapM8,11198
-datachain-0.9.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-datachain-0.9.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.9.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.9.0.dist-info/RECORD,,
+datachain-0.10.0.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.10.0.dist-info/METADATA,sha256=4Eoe6lnoy_HBYtdzrAIjNnagKXagattQ_mluP9WC-ek,11195
+datachain-0.10.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+datachain-0.10.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.10.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.10.0.dist-info/RECORD,,