datachain 0.9.1__tar.gz → 0.11.0__tar.gz

{datachain-0.9.1 → datachain-0.11.0}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.9.6'
+    rev: 'v0.9.7'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.9.1 → datachain-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.9.1
+Version: 0.11.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -49,6 +49,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
 Requires-Dist: websockets
+Requires-Dist: tomli; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -102,7 +103,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.74; extra == "examples"
+Requires-Dist: ultralytics==8.3.78; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================
@@ -175,7 +176,7 @@ high confidence scores.
 
     likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
                                    & (Column("meta.inference.class_") == "cat"))
-    likely_cats.export_files("high-confidence-cats/", signal="file")
+    likely_cats.to_storage("high-confidence-cats/", signal="file")
 
 
 Example: LLM based text-file evaluation
@@ -216,7 +217,7 @@ Python code:
     )
 
     successful_chain = chain.filter(Column("is_success") == True)
-    successful_chain.export_files("./output_mistral")
+    successful_chain.to_storage("./output_mistral")
 
     print(f"{successful_chain.count()} files were exported")
 

{datachain-0.9.1 → datachain-0.11.0}/README.rst

@@ -68,7 +68,7 @@ high confidence scores.
 
     likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
                                    & (Column("meta.inference.class_") == "cat"))
-    likely_cats.export_files("high-confidence-cats/", signal="file")
+    likely_cats.to_storage("high-confidence-cats/", signal="file")
 
 
 Example: LLM based text-file evaluation
@@ -109,7 +109,7 @@ Python code:
     )
 
     successful_chain = chain.filter(Column("is_success") == True)
-    successful_chain.export_files("./output_mistral")
+    successful_chain.to_storage("./output_mistral")
 
     print(f"{successful_chain.count()} files were exported")
 

{datachain-0.9.1 → datachain-0.11.0}/docs/quick-start.md

@@ -47,7 +47,7 @@ annotated = images_id.merge(meta, on="id", right_on="meta.id")
 
 likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
                                & (Column("meta.inference.class_") == "cat"))
-likely_cats.export_files("high-confidence-cats/", signal="file")
+likely_cats.to_storage("high-confidence-cats/", signal="file")
 ```
 
 ## Data curation with a local AI model
@@ -85,7 +85,7 @@ chain = (
 )
 
 positive_chain = chain.filter(Column("is_positive") == True)
-positive_chain.export_files("./output")
+positive_chain.to_storage("./output")
 
 print(f"{positive_chain.count()} files were exported")
 ```

{datachain-0.9.1 → datachain-0.11.0}/pyproject.toml

@@ -51,7 +51,8 @@ dependencies = [
   "platformdirs",
   "dvc-studio-client>=0.21,<1",
   "tabulate",
-  "websockets"
+  "websockets",
+  "tomli;python_version<'3.11'"
 ]
 
 [project.optional-dependencies]
@@ -118,7 +119,7 @@ examples = [
   "defusedxml",
   "accelerate",
   "huggingface_hub[hf_transfer]",
-  "ultralytics==8.3.74",
+  "ultralytics==8.3.78",
   "open_clip_torch"
 ]
 

{datachain-0.9.1 → datachain-0.11.0}/src/datachain/client/fsspec.py

@@ -390,7 +390,7 @@ class Client(ABC):
         )  # type: ignore[return-value]
 
     def upload(self, data: bytes, path: str) -> "File":
-        full_path = self.get_full_path(path)
+        full_path = path if path.startswith(self.PREFIX) else self.get_full_path(path)
 
         parent = posixpath.dirname(full_path)
         self.fs.makedirs(parent, exist_ok=True)

{datachain-0.9.1 → datachain-0.11.0}/src/datachain/lib/dc.py

@@ -411,6 +411,7 @@ class DataChain:
         object_name: str = "file",
         update: bool = False,
         anon: bool = False,
+        client_config: Optional[dict] = None,
     ) -> "Self":
         """Get data from a storage as a list of file with all file attributes.
         It returns the chain itself as usual.
@@ -423,15 +424,32 @@ class DataChain:
             object_name : Created object column name.
             update : force storage reindexing. Default is False.
             anon : If True, we will treat cloud bucket as public one
+            client_config : Optional client configuration for the storage client.
 
         Example:
+            Simple call from s3
             ```py
             chain = DataChain.from_storage("s3://my-bucket/my-dir")
             ```
+
+            With AWS S3-compatible storage
+            ```py
+            chain = DataChain.from_storage(
+                "s3://my-bucket/my-dir",
+                client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
+            )
+            ```
+
+            Pass existing session
+            ```py
+            session = Session.get()
+            chain = DataChain.from_storage("s3://my-bucket/my-dir", session=session)
+            ```
         """
         file_type = get_file_type(type)
 
-        client_config = {"anon": True} if anon else None
+        if anon:
+            client_config = (client_config or {}) | {"anon": True}
         session = Session.get(session, client_config=client_config, in_memory=in_memory)
         cache = session.catalog.cache
         client_config = session.catalog.client_config
@@ -481,25 +499,56 @@ class DataChain:
         version: Optional[int] = None,
         session: Optional[Session] = None,
         settings: Optional[dict] = None,
-        fallback_to_remote: bool = True,
+        fallback_to_studio: bool = True,
     ) -> "Self":
         """Get data from a saved Dataset. It returns the chain itself.
+        If dataset or version is not found locally, it will try to pull it from Studio.
 
         Parameters:
             name : dataset name
             version : dataset version
+            session : Session to use for the chain.
+            settings : Settings to use for the chain.
+            fallback_to_studio : Try to pull dataset from Studio if not found locally.
+                Default is True.
 
         Example:
             ```py
             chain = DataChain.from_dataset("my_cats")
             ```
+
+            ```py
+            chain = DataChain.from_dataset("my_cats", fallback_to_studio=False)
+            ```
+
+            ```py
+            chain = DataChain.from_dataset("my_cats", version=1)
+            ```
+
+            ```py
+            session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
+            settings = {
+                "cache": True,
+                "parallel": 4,
+                "workers": 4,
+                "min_task_size": 1000,
+                "prefetch": 10,
+            }
+            chain = DataChain.from_dataset(
+                name="my_cats",
+                version=1,
+                session=session,
+                settings=settings,
+                fallback_to_studio=True,
+            )
+            ```
         """
         query = DatasetQuery(
             name=name,
             version=version,
             session=session,
             indexing_column_types=File._datachain_column_types,
-            fallback_to_remote=fallback_to_remote,
+            fallback_to_studio=fallback_to_studio,
         )
         telemetry.send_event_once("class", "datachain_init", name=name, version=version)
         if settings:
@@ -2444,7 +2493,7 @@ class DataChain:
         self._setup = self._setup | kwargs
         return self
 
-    def export_files(
+    def to_storage(
         self,
         output: str,
         signal: str = "file",
@@ -2462,6 +2511,13 @@ class DataChain:
             use_cache: If `True`, cache the files before exporting.
             link_type: Method to use for exporting files.
                 Falls back to `'copy'` if symlinking fails.
+
+        Example:
+            Cross cloud transfer
+            ```py
+            ds = DataChain.from_storage("s3://mybucket")
+            ds.to_storage("gs://mybucket", placement="filename")
+            ```
         """
         if placement == "filename" and (
             self._query.distinct(pathfunc.name(C(f"{signal}__path"))).count()

{datachain-0.9.1 → datachain-0.11.0}/src/datachain/lib/file.py

@@ -17,6 +17,7 @@ from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
 
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
+from fsspec.utils import stringify_path
 from PIL import Image as PilImage
 from pydantic import Field, field_validator
 
@@ -270,8 +271,13 @@ class File(DataModel):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        with open(destination, mode="wb") as f:
-            f.write(self.read())
+        destination = stringify_path(destination)
+        client: Client = self._catalog.get_client(destination)
+
+        if client.PREFIX == "file://" and not destination.startswith(client.PREFIX):
+            destination = Path(destination).absolute().as_uri()
+
+        client.upload(self.read(), destination)
 
     def _symlink_to(self, destination: str):
         if self.location:
@@ -285,6 +291,7 @@ class File(DataModel):
             source = self.get_path()
         else:
             raise OSError(errno.EXDEV, "can't link across filesystems")
+
         return os.symlink(source, destination)
 
     def export(
@@ -299,7 +306,8 @@ class File(DataModel):
             self._caching_enabled = use_cache
         dst = self.get_destination_path(output, placement)
         dst_dir = os.path.dirname(dst)
-        os.makedirs(dst_dir, exist_ok=True)
+        client: Client = self._catalog.get_client(dst_dir)
+        client.fs.makedirs(dst_dir, exist_ok=True)
 
         if link_type == "symlink":
             try:
@@ -496,7 +504,10 @@ class TextFile(File):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        with open(destination, mode="w") as f:
+        destination = stringify_path(destination)
+
+        client: Client = self._catalog.get_client(destination)
+        with client.fs.open(destination, mode="w") as f:
             f.write(self.read_text())
 
 
@@ -510,7 +521,11 @@ class ImageFile(File):
 
     def save(self, destination: str):
         """Writes it's content to destination"""
-        self.read().save(destination)
+        destination = stringify_path(destination)
+
+        client: Client = self._catalog.get_client(destination)
+        with client.fs.open(destination, mode="wb") as f:
+            self.read().save(f)
 
 
 class Image(DataModel):

{datachain-0.9.1 → datachain-0.11.0}/src/datachain/query/dataset.py

@@ -1085,7 +1085,7 @@ class DatasetQuery:
         session: Optional[Session] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
         in_memory: bool = False,
-        fallback_to_remote: bool = True,
+        fallback_to_studio: bool = True,
     ) -> None:
         self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
@@ -1103,7 +1103,7 @@ class DatasetQuery:
 
         self.name = name
 
-        if fallback_to_remote and is_token_set():
+        if fallback_to_studio and is_token_set():
             ds = self.catalog.get_dataset_with_remote_fallback(name, version)
         else:
             ds = self.catalog.get_dataset(name)

{datachain-0.9.1 → datachain-0.11.0}/src/datachain/query/session.py

@@ -139,21 +139,33 @@ class Session:
 
         # Access the active (most recent) context from the stack
         if cls.SESSION_CONTEXTS:
-            return cls.SESSION_CONTEXTS[-1]
+            session = cls.SESSION_CONTEXTS[-1]
 
-        if cls.GLOBAL_SESSION_CTX is None:
+        elif cls.GLOBAL_SESSION_CTX is None:
             cls.GLOBAL_SESSION_CTX = Session(
                 cls.GLOBAL_SESSION_NAME,
                 catalog,
                 client_config=client_config,
                 in_memory=in_memory,
             )
+            session = cls.GLOBAL_SESSION_CTX
 
             atexit.register(cls._global_cleanup)
             cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
             sys.excepthook = cls.except_hook
+        else:
+            session = cls.GLOBAL_SESSION_CTX
 
-        return cls.GLOBAL_SESSION_CTX
+        if client_config and session.catalog.client_config != client_config:
+            session = Session(
+                "session" + uuid4().hex[:4],
+                catalog,
+                client_config=client_config,
+                in_memory=in_memory,
+            )
+            session.__enter__()
+
+        return session
 
     @staticmethod
     def except_hook(exc_type, exc_value, exc_traceback):

datachain-0.11.0/src/datachain/script_meta.py

@@ -0,0 +1,147 @@
+import re
+from dataclasses import dataclass
+from typing import Any, Optional
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    # tomllib is in standard library from python 3.11 so for earlier versions
+    # we need tomli
+    import tomli as tomllib  # type: ignore[no-redef]
+
+
+class ScriptConfigParsingError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+@dataclass
+class ScriptConfig:
+    """
+    Class that is parsing inline script metadata to get some basic information for
+    running datachain script like python version, dependencies, attachments etc.
+    Inline script metadata must follow the format described in https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata.
+    Example of script with inline metadata:
+        # /// script
+        # requires-python = ">=3.12"
+        #
+        # dependencies = [
+        #   "pandas < 2.1.0",
+        #   "numpy == 1.26.4"
+        # ]
+        #
+        # [tools.datachain.workers]
+        # num_workers = 3
+        #
+        # [tools.datachain.attachments]
+        # image1 = "s3://ldb-public/image1.jpg"
+        # file1 = "s3://ldb-public/file.pdf"
+        #
+        # [tools.datachain.params]
+        # min_length_sec = 1
+        # cache = false
+        #
+        # [tools.datachain.inputs]
+        # threshold = 0.5
+        # start_ds_name = "ds://start"
+        #
+        # [tools.datachain.outputs]
+        # result_dataset = "ds://res"
+        # result_dir = "/temp"
+        #
+        # ///
+
+        import sys
+        import pandas as pd
+
+        print(f"Python version: {sys.version_info}")
+        print(f"Pandas version: {pd.__version__}")
+
+    """
+
+    python_version: Optional[str]
+    dependencies: list[str]
+    attachments: dict[str, str]
+    params: dict[str, Any]
+    inputs: dict[str, Any]
+    outputs: dict[str, Any]
+    num_workers: Optional[int] = None
+
+    def __init__(
+        self,
+        python_version: Optional[str] = None,
+        dependencies: Optional[list[str]] = None,
+        attachments: Optional[dict[str, str]] = None,
+        params: Optional[dict[str, Any]] = None,
+        inputs: Optional[dict[str, Any]] = None,
+        outputs: Optional[dict[str, Any]] = None,
+        num_workers: Optional[int] = None,
+    ):
+        self.python_version = python_version
+        self.dependencies = dependencies or []
+        self.attachments = attachments or {}
+        self.params = params or {}
+        self.inputs = inputs or {}
+        self.outputs = outputs or {}
+        self.num_workers = num_workers
+
+    def get_param(self, name: str, default: Any) -> Any:
+        return self.params.get(name, default)
+
+    def get_input(self, name: str, default: Any) -> Any:
+        return self.inputs.get(name, default)
+
+    def get_output(self, name: str, default: Any) -> Any:
+        return self.outputs.get(name, default)
+
+    def get_attachment(self, name: str, default: Any) -> Any:
+        return self.attachments.get(name, default)
+
+    @staticmethod
+    def read(script: str) -> Optional[dict]:
+        """Converts inline script metadata to dict with all found data"""
+        regex = (
+            r"(?m)^# \/\/\/ (?P<type>[a-zA-Z0-9-]+)[ \t]*$[\r\n|\r|\n]"
+            "(?P<content>(?:^#(?:| .*)$[\r\n|\r|\n])+)^# \\/\\/\\/[ \t]*$"
+        )
+        name = "script"
+        matches = list(
+            filter(lambda m: m.group("type") == name, re.finditer(regex, script))
+        )
+        if len(matches) > 1:
+            raise ValueError(f"Multiple {name} blocks found")
+        if len(matches) == 1:
+            content = "".join(
+                line[2:] if line.startswith("# ") else line[1:]
+                for line in matches[0].group("content").splitlines(keepends=True)
+            )
+            return tomllib.loads(content)
+        return None
+
+    @staticmethod
+    def parse(script: str) -> Optional["ScriptConfig"]:
+        """
+        Method that is parsing inline script metadata from datachain script and
+        instantiating ScriptConfig class with found data. If no inline metadata is
+        found, it returns None
+        """
+        try:
+            meta = ScriptConfig.read(script)
+            if not meta:
+                return None
+            custom = meta.get("tools", {}).get("datachain", {})
+            return ScriptConfig(
+                python_version=meta.get("requires-python"),
+                dependencies=meta.get("dependencies"),
+                num_workers=custom.get("workers", {}).get("num_workers"),
+                attachments=custom.get("attachments"),
+                params={k: str(v) for k, v in custom.get("params").items()}
+                if custom.get("params")
+                else None,
+                inputs=custom.get("inputs"),
+                outputs=custom.get("outputs"),
+            )
+        except Exception as e:
+            raise ScriptConfigParsingError(
+                f"Error when parsing script meta: {e}"
+            ) from e

{datachain-0.9.1 → datachain-0.11.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.9.1
+Version: 0.11.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -49,6 +49,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
 Requires-Dist: websockets
+Requires-Dist: tomli; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -102,7 +103,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.74; extra == "examples"
+Requires-Dist: ultralytics==8.3.78; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================
@@ -175,7 +176,7 @@ high confidence scores.
 
     likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
                                    & (Column("meta.inference.class_") == "cat"))
-    likely_cats.export_files("high-confidence-cats/", signal="file")
+    likely_cats.to_storage("high-confidence-cats/", signal="file")
 
 
 Example: LLM based text-file evaluation
@@ -216,7 +217,7 @@ Python code:
     )
 
     successful_chain = chain.filter(Column("is_success") == True)
-    successful_chain.export_files("./output_mistral")
+    successful_chain.to_storage("./output_mistral")
 
     print(f"{successful_chain.count()} files were exported")
 

{datachain-0.9.1 → datachain-0.11.0}/src/datachain.egg-info/SOURCES.txt

@@ -77,6 +77,7 @@ src/datachain/nodes_fetcher.py
 src/datachain/nodes_thread_pool.py
 src/datachain/progress.py
 src/datachain/py.typed
+src/datachain/script_meta.py
 src/datachain/studio.py
 src/datachain/telemetry.py
 src/datachain/utils.py
@@ -232,6 +233,7 @@ tests/func/__init__.py
 tests/func/fake-service-account-credentials.json
 tests/func/test_catalog.py
 tests/func/test_client.py
+tests/func/test_cloud_transfer.py
 tests/func/test_data_storage.py
 tests/func/test_datachain.py
 tests/func/test_datachain_merge.py
@@ -278,6 +280,7 @@ tests/unit/test_pytorch.py
 tests/unit/test_query.py
 tests/unit/test_query_metrics.py
 tests/unit/test_query_params.py
+tests/unit/test_script_meta.py
 tests/unit/test_serializer.py
 tests/unit/test_session.py
 tests/unit/test_utils.py

{datachain-0.9.1 → datachain-0.11.0}/src/datachain.egg-info/requires.txt

@@ -32,6 +32,9 @@ dvc-studio-client<1,>=0.21
 tabulate
 websockets
 
+[:python_version < "3.11"]
+tomli
+
 [dev]
 datachain[docs,tests]
 mypy==1.15.0
@@ -55,7 +58,7 @@ datachain[tests]
 defusedxml
 accelerate
 huggingface_hub[hf_transfer]
-ultralytics==8.3.74
+ultralytics==8.3.78
 open_clip_torch
 
 [hf]

{datachain-0.9.1 → datachain-0.11.0}/tests/conftest.py

@@ -472,9 +472,9 @@ def cloud_server_credentials(cloud_server, monkeypatch):
 
 def get_cloud_test_catalog(cloud_server, tmp_path, metastore, warehouse):
     cache_dir = tmp_path / ".datachain" / "cache"
-    cache_dir.mkdir(parents=True)
+    cache_dir.mkdir(parents=True, exist_ok=True)
     tmpfile_dir = tmp_path / ".datachain" / "tmp"
-    tmpfile_dir.mkdir()
+    tmpfile_dir.mkdir(exist_ok=True)
 
     catalog = Catalog(
         metastore=metastore,

datachain-0.11.0/tests/func/test_cloud_transfer.py

@@ -0,0 +1,68 @@
+import pytest
+
+from datachain import Session
+from datachain.lib.dc import DataChain
+from tests.conftest import get_cloud_test_catalog, make_cloud_server
+
+
+def test_cross_cloud_transfer(
+    request,
+    tmp_upath_factory,
+    tree,
+    tmp_path,
+    metastore,
+    warehouse,
+):
+    disabled_remotes = request.config.getoption("--disable-remotes") or []
+
+    if any(remote in disabled_remotes for remote in ["azure", "gs", "all"]):
+        pytest.skip("Skipping all tests for azure, gs or all remotes")
+
+    azure_path = tmp_upath_factory.mktemp("azure", version_aware=False)
+    azure_server = make_cloud_server(azure_path, "azure", tree)
+
+    gcloud_path = tmp_upath_factory.mktemp("gs", version_aware=False)
+    gcloud_server = make_cloud_server(gcloud_path, "gs", tree)
+
+    # Initialize cloud catalogs
+    azure_catalog = get_cloud_test_catalog(azure_server, tmp_path, metastore, warehouse)
+    gcloud_catalog = get_cloud_test_catalog(
+        gcloud_server, tmp_path, metastore, warehouse
+    )
+
+    # Define test file paths
+    test_filename = "image_1.jpg"
+    test_content = b"bytes"
+
+    source_dir = f"{azure_catalog.src_uri}/source-test-images"
+    source_file = f"{source_dir}/{test_filename}"
+
+    dest_dir = f"{gcloud_catalog.src_uri}/destination-test-images"
+    dest_file = f"{dest_dir}/{test_filename}"
+
+    # Get cloud clients
+    azure_client = azure_catalog.catalog.get_client(source_file)
+    gcloud_client = gcloud_catalog.catalog.get_client(dest_file)
+
+    try:
+        # Create test file in Azure
+        with azure_client.fs.open(source_file, "wb") as f:
+            f.write(test_content)
+
+        # Perform cross-cloud transfer
+        combined_config = azure_server.client_config | gcloud_server.client_config
+        with Session("testSession", client_config=combined_config):
+            datachain = DataChain.from_storage(source_dir)
+            datachain.to_storage(dest_dir, placement="filename")
+
+        # Verify transfer
+        with gcloud_client.fs.open(dest_file, "rb") as f:
+            assert f.read() == test_content
+
+    finally:
+        # Cleanup
+        try:
+            azure_client.fs.rm(source_dir, recursive=True)
+            gcloud_client.fs.rm(dest_dir, recursive=True)
+        except FileNotFoundError:
+            pass