flyte 2.0.0b22__py3-none-any.whl → 2.0.0b30__py3-none-any.whl

flyte/io/_dir.py

@@ -4,12 +4,14 @@ import os
 from pathlib import Path
 from typing import AsyncIterator, Dict, Generic, Iterator, List, Optional, Type, TypeVar, Union
 
-from flyteidl.core import literals_pb2, types_pb2
+from flyteidl2.core import literals_pb2, types_pb2
 from fsspec.asyn import AsyncFileSystem
+from fsspec.utils import get_protocol
 from mashumaro.types import SerializableType
 from pydantic import BaseModel, model_validator
 
 import flyte.storage as storage
+from flyte._context import internal_ctx
 from flyte.io._file import File
 from flyte.types import TypeEngine, TypeTransformer, TypeTransformerFailedError
 
@@ -20,28 +22,176 @@ T = TypeVar("T")
 class Dir(BaseModel, Generic[T], SerializableType):
     """
     A generic directory class representing a directory with files of a specified format.
-    Provides both async and sync interfaces for directory operations.
-    Users are responsible for handling all I/O - the type transformer for Dir does not do any automatic uploading
-    or downloading of files.
+    Provides both async and sync interfaces for directory operations. All methods without _sync suffix are async.
+
+    The class should be instantiated using one of the class methods. The constructor should only be used to
+    instantiate references to existing remote directories.
 
     The generic type T represents the format of the files in the directory.
 
-    Example:
+    Important methods:
+    - `from_existing_remote`: Create a Dir object referencing an existing remote directory.
+    - `from_local` / `from_local_sync`: Upload a local directory to remote storage.
+
+    **Asynchronous methods**:
+    - `walk`: Asynchronously iterate through files in the directory.
+    - `list_files`: Asynchronously get a list of all files (non-recursive).
+    - `download`: Asynchronously download the entire directory to a local path.
+    - `exists`: Asynchronously check if the directory exists.
+    - `get_file`: Asynchronously get a specific file from the directory by name.
+
+    **Synchronous methods** (suffixed with `_sync`):
+    - `walk_sync`: Synchronously iterate through files in the directory.
+    - `list_files_sync`: Synchronously get a list of all files (non-recursive).
+    - `download_sync`: Synchronously download the entire directory to a local path.
+    - `exists_sync`: Synchronously check if the directory exists.
+    - `get_file_sync`: Synchronously get a specific file from the directory by name.
+
+    Example: Walk through directory files recursively (Async).
+
+    ```python
+    @env.task
+    async def process_all_files(d: Dir) -> int:
+        file_count = 0
+        async for file in d.walk(recursive=True):
+            async with file.open("rb") as f:
+                content = await f.read()
+                # Process content
+                file_count += 1
+        return file_count
+    ```
+
+    Example: Walk through directory files recursively (Sync).
+
+    ```python
+    @env.task
+    def process_all_files_sync(d: Dir) -> int:
+        file_count = 0
+        for file in d.walk_sync(recursive=True):
+            with file.open_sync("rb") as f:
+                content = f.read()
+                # Process content
+                file_count += 1
+        return file_count
+    ```
+
+    Example: List files in directory (Async).
+
+    ```python
+    @env.task
+    async def count_files(d: Dir) -> int:
+        files = await d.list_files()
+        return len(files)
+    ```
+
+    Example: List files in directory (Sync).
+
+    ```python
+    @env.task
+    def count_files_sync(d: Dir) -> int:
+        files = d.list_files_sync()
+        return len(files)
+    ```
+
+    Example: Get a specific file from directory (Async).
+
     ```python
-    # Async usage
-    from pandas import DataFrame
-    data_dir = Dir[DataFrame](path="s3://my-bucket/data/")
-
-    # Walk through files
-    async for file in data_dir.walk():
-        async with file.open() as f:
-            content = await f.read()
-
-    # Sync alternative
-    for file in data_dir.walk_sync():
-        with file.open_sync() as f:
-            content = f.read()
+    @env.task
+    async def read_config_file(d: Dir) -> str:
+        config_file = await d.get_file("config.json")
+        if config_file:
+            async with config_file.open("rb") as f:
+                return (await f.read()).decode("utf-8")
+        return "Config not found"
     ```
+
+    Example: Get a specific file from directory (Sync).
+
+    ```python
+    @env.task
+    def read_config_file_sync(d: Dir) -> str:
+        config_file = d.get_file_sync("config.json")
+        if config_file:
+            with config_file.open_sync("rb") as f:
+                return f.read().decode("utf-8")
+        return "Config not found"
+    ```
+
+    Example: Upload a local directory to remote storage (Async).
+
+    ```python
+    @env.task
+    async def upload_directory() -> Dir:
+        # Create local directory with files
+        os.makedirs("/tmp/my_data", exist_ok=True)
+        with open("/tmp/my_data/file1.txt", "w") as f:
+            f.write("data1")
+        # Upload to remote storage
+        return await Dir.from_local("/tmp/my_data/")
+    ```
+
+    Example: Upload a local directory to remote storage (Sync).
+
+    ```python
+    @env.task
+    def upload_directory_sync() -> Dir:
+        # Create local directory with files
+        os.makedirs("/tmp/my_data", exist_ok=True)
+        with open("/tmp/my_data/file1.txt", "w") as f:
+            f.write("data1")
+        # Upload to remote storage
+        return Dir.from_local_sync("/tmp/my_data/")
+    ```
+
+    Example: Download a directory to local storage (Async).
+
+    ```python
+    @env.task
+    async def download_directory(d: Dir) -> str:
+        local_path = await d.download()
+        # Process files in local directory
+        return local_path
+    ```
+
+    Example: Download a directory to local storage (Sync).
+
+    ```python
+    @env.task
+    def download_directory_sync(d: Dir) -> str:
+        local_path = d.download_sync()
+        # Process files in local directory
+        return local_path
+    ```
+
+    Example: Reference an existing remote directory.
+
+    ```python
+    @env.task
+    async def process_existing_dir() -> int:
+        d = Dir.from_existing_remote("s3://my-bucket/data/")
+        files = await d.list_files()
+        return len(files)
+    ```
+
+    Example: Check if directory exists (Async).
+
+    ```python
+    @env.task
+    async def check_directory(d: Dir) -> bool:
+        return await d.exists()
+    ```
+
+    Example: Check if directory exists (Sync).
+
+    ```python
+    @env.task
+    def check_directory_sync(d: Dir) -> bool:
+        return d.exists_sync()
+    ```
+
+    Args:
+        path: The path to the directory (can be local or remote)
+        name: Optional name for the directory (defaults to basename of path)
     """
 
     # Represents either a local or remote path.
@@ -56,20 +206,24 @@ class Dir(BaseModel, Generic[T], SerializableType):
     @model_validator(mode="before")
     @classmethod
     def pre_init(cls, data):
+        """Internal: Pydantic validator to set default name from path. Not intended for direct use."""
         if data.get("name") is None:
             data["name"] = Path(data["path"]).name
         return data
 
     def _serialize(self) -> Dict[str, Optional[str]]:
+        """Internal: Serialize Dir to dictionary. Not intended for direct use."""
         pyd_dump = self.model_dump()
         return pyd_dump
 
     @classmethod
     def _deserialize(cls, file_dump: Dict[str, Optional[str]]) -> Dir:
+        """Internal: Deserialize Dir from dictionary. Not intended for direct use."""
         return cls.model_validate(file_dump)
 
     @classmethod
     def schema_match(cls, incoming: dict):
+        """Internal: Check if incoming schema matches Dir schema. Not intended for direct use."""
         this_schema = cls.model_json_schema()
         current_required = this_schema.get("required")
         incoming_required = incoming.get("required")
@@ -86,19 +240,48 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Asynchronously walk through the directory and yield File objects.
 
-        Args:
-            recursive: If True, recursively walk subdirectories
-            max_depth: Maximum depth for recursive walking
+        Use this to iterate through all files in a directory. Each yielded File can be read directly without
+        downloading.
 
-        Yields:
-            File objects for each file found in the directory
+        Example (Async - Recursive):
+
+        ```python
+        @env.task
+        async def list_all_files(d: Dir) -> list[str]:
+            file_names = []
+            async for file in d.walk(recursive=True):
+                file_names.append(file.name)
+            return file_names
+        ```
+
+        Example (Async - Non-recursive):
 
-        Example:
         ```python
-        async for file in directory.walk():
-            local_path = await file.download()
-            # Process the file
+        @env.task
+        async def list_top_level_files(d: Dir) -> list[str]:
+            file_names = []
+            async for file in d.walk(recursive=False):
+                file_names.append(file.name)
+            return file_names
         ```
+
+        Example (Async - With max depth):
+
+        ```python
+        @env.task
+        async def list_files_max_depth(d: Dir) -> list[str]:
+            file_names = []
+            async for file in d.walk(recursive=True, max_depth=2):
+                file_names.append(file.name)
+            return file_names
+        ```
+
+        Args:
+            recursive: If True, recursively walk subdirectories. If False, only list files in the top-level directory.
+            max_depth: Maximum depth for recursive walking. If None, walk through all subdirectories.
+
+        Yields:
+            File objects for each file found in the directory
         """
         fs = storage.get_underlying_filesystem(path=self.path)
         if recursive is False:
@@ -125,20 +308,48 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Synchronously walk through the directory and yield File objects.
 
-        Args:
-            recursive: If True, recursively walk subdirectories
-            file_pattern: Glob pattern to filter files
-            max_depth: Maximum depth for recursive walking
+        Use this in non-async tasks to iterate through all files in a directory.
 
-        Yields:
-            File objects for each file found in the directory
+        Example (Sync - Recursive):
+
+        ```python
+        @env.task
+        def list_all_files_sync(d: Dir) -> list[str]:
+            file_names = []
+            for file in d.walk_sync(recursive=True):
+                file_names.append(file.name)
+            return file_names
+        ```
+
+        Example (Sync - With file pattern):
+
+        ```python
+        @env.task
+        def list_text_files(d: Dir) -> list[str]:
+            file_names = []
+            for file in d.walk_sync(recursive=True, file_pattern="*.txt"):
+                file_names.append(file.name)
+            return file_names
+        ```
+
+        Example (Sync - Non-recursive with max depth):
 
-        Example:
         ```python
-        for file in directory.walk_sync():
-            local_path = file.download_sync()
-            # Process the file
+        @env.task
+        def list_files_limited(d: Dir) -> list[str]:
+            file_names = []
+            for file in d.walk_sync(recursive=True, max_depth=2):
+                file_names.append(file.name)
+            return file_names
         ```
+
+        Args:
+            recursive: If True, recursively walk subdirectories. If False, only list files in the top-level directory.
+            file_pattern: Glob pattern to filter files (e.g., "*.txt", "*.csv"). Default is "*" (all files).
+            max_depth: Maximum depth for recursive walking. If None, walk through all subdirectories.
+
+        Yields:
+            File objects for each file found in the directory
         """
         fs = storage.get_underlying_filesystem(path=self.path)
         for parent, _, files in fs.walk(self.path, maxdepth=max_depth):
@@ -153,14 +364,32 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Asynchronously get a list of all files in the directory (non-recursive).
 
+        Use this when you need a list of all files in the top-level directory at once.
+
         Returns:
-            A list of File objects
+            A list of File objects for files in the top-level directory
+
+        Example (Async):
 
-        Example:
         ```python
-        files = await directory.list_files()
-        for file in files:
-            # Process the file
+        @env.task
+        async def count_files(d: Dir) -> int:
+            files = await d.list_files()
+            return len(files)
+        ```
+
+        Example (Async - Process files):
+
+        ```python
+        @env.task
+        async def process_all_files(d: Dir) -> list[str]:
+            files = await d.list_files()
+            contents = []
+            for file in files:
+                async with file.open("rb") as f:
+                    content = await f.read()
+                    contents.append(content.decode("utf-8"))
+            return contents
         ```
         """
         # todo: this should probably also just defer to fsspec.find()
@@ -173,14 +402,32 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Synchronously get a list of all files in the directory (non-recursive).
 
+        Use this in non-async tasks when you need a list of all files in the top-level directory at once.
+
         Returns:
-            A list of File objects
+            A list of File objects for files in the top-level directory
+
+        Example (Sync):
 
-        Example:
         ```python
-        files = directory.list_files_sync()
-        for file in files:
-            # Process the file
+        @env.task
+        def count_files_sync(d: Dir) -> int:
+            files = d.list_files_sync()
+            return len(files)
+        ```
+
+        Example (Sync - Process files):
+
+        ```python
+        @env.task
+        def process_all_files_sync(d: Dir) -> list[str]:
+            files = d.list_files_sync()
+            contents = []
+            for file in files:
+                with file.open_sync("rb") as f:
+                    content = f.read()
+                    contents.append(content.decode("utf-8"))
+            return contents
         ```
         """
         return list(self.walk_sync(recursive=False))
@@ -189,19 +436,43 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Asynchronously download the entire directory to a local path.
 
-        Args:
-            local_path: The local path to download the directory to. If None, a temporary
-                       directory will be used.
+        Use this when you need to download all files in a directory to your local filesystem for processing.
 
-        Returns:
-            The path to the downloaded directory
+        Example (Async):
+
+        ```python
+        @env.task
+        async def download_directory(d: Dir) -> str:
+            local_dir = await d.download()
+            # Process files in the local directory
+            return local_dir
+        ```
+
+        Example (Async - Download to specific path):
 
-        Example:
         ```python
-        local_dir = await directory.download('/tmp/my_data/')
+        @env.task
+        async def download_to_path(d: Dir) -> str:
+            local_dir = await d.download("/tmp/my_data/")
+            return local_dir
         ```
+
+        Args:
+            local_path: The local path to download the directory to. If None, a temporary
+                       directory will be used and a path will be generated.
+
+        Returns:
+            The absolute path to the downloaded directory
         """
-        local_dest = str(local_path) if local_path else str(storage.get_random_local_path())
+        # If no local_path specified, create a unique path + append source directory name
+        if local_path is None:
+            unique_path = storage.get_random_local_path()
+            source_dirname = Path(self.path).name  # will need to be updated for windows
+            local_dest = str(Path(unique_path) / source_dirname)
+        else:
+            # If local_path is specified, use it directly (contents go into it)
+            local_dest = str(local_path)
+
         if not storage.is_remote(self.path):
             if not local_path or local_path == self.path:
                 # Skip copying
@@ -216,25 +487,49 @@ class Dir(BaseModel, Generic[T], SerializableType):
                     await loop.run_in_executor(None, lambda: shutil.copytree(self.path, local_dest, dirs_exist_ok=True))
 
                 await copy_tree()
+                return local_dest
         return await storage.get(self.path, local_dest, recursive=True)
 
     def download_sync(self, local_path: Optional[Union[str, Path]] = None) -> str:
         """
         Synchronously download the entire directory to a local path.
 
-        Args:
-            local_path: The local path to download the directory to. If None, a temporary
-                       directory will be used.
+        Use this in non-async tasks when you need to download all files in a directory to your local filesystem.
 
-        Returns:
-            The path to the downloaded directory
+        Example (Sync):
+
+        ```python
+        @env.task
+        def download_directory_sync(d: Dir) -> str:
+            local_dir = d.download_sync()
+            # Process files in the local directory
+            return local_dir
+        ```
+
+        Example (Sync - Download to specific path):
 
-        Example:
         ```python
-        local_dir = directory.download_sync('/tmp/my_data/')
+        @env.task
+        def download_to_path_sync(d: Dir) -> str:
+            local_dir = d.download_sync("/tmp/my_data/")
+            return local_dir
         ```
+        Args:
+            local_path: The local path to download the directory to. If None, a temporary
+                       directory will be used and a path will be generated.
+
+        Returns:
+            The absolute path to the downloaded directory
         """
-        local_dest = str(local_path) if local_path else str(storage.get_random_local_path())
+        # If no local_path specified, create a unique path + append source directory name
+        if local_path is None:
+            unique_path = storage.get_random_local_path()
+            source_dirname = Path(self.path).name
+            local_dest = str(Path(unique_path) / source_dirname)
+        else:
+            # If local_path is specified, use it directly (contents go into it)
+            local_dest = str(local_path)
+
         if not storage.is_remote(self.path):
             if not local_path or local_path == self.path:
                 # Skip copying
@@ -244,80 +539,188 @@ class Dir(BaseModel, Generic[T], SerializableType):
                 import shutil
 
                 shutil.copytree(self.path, local_dest, dirs_exist_ok=True)
+                return local_dest
 
-        # Figure this out when we figure out the final sync story
-        raise NotImplementedError("Sync download is not implemented for remote paths")
+        fs = storage.get_underlying_filesystem(path=self.path)
+        fs.get(self.path, local_dest, recursive=True)
+        return local_dest
 
     @classmethod
     async def from_local(
         cls,
         local_path: Union[str, Path],
-        remote_path: Optional[str] = None,
+        remote_destination: Optional[str] = None,
         dir_cache_key: Optional[str] = None,
     ) -> Dir[T]:
         """
-        Asynchronously create a new Dir by uploading a local directory to the configured remote store.
+        Asynchronously create a new Dir by uploading a local directory to remote storage.
+
+        Use this in async tasks when you have a local directory that needs to be uploaded to remote storage.
 
+        Example (Async):
+
+        ```python
+        @env.task
+        async def upload_local_directory() -> Dir:
+            # Create a local directory with files
+            os.makedirs("/tmp/data_dir", exist_ok=True)
+            with open("/tmp/data_dir/file1.txt", "w") as f:
+                f.write("data1")
+
+            # Upload to remote storage
+            remote_dir = await Dir.from_local("/tmp/data_dir/")
+            return remote_dir
+        ```
+
+        Example (Async - With specific destination):
+
+        ```python
+        @env.task
+        async def upload_to_specific_path() -> Dir:
+            remote_dir = await Dir.from_local("/tmp/data_dir/", "s3://my-bucket/data/")
+            return remote_dir
+        ```
+
+        Example (Async - With cache key):
+
+        ```python
+        @env.task
+        async def upload_with_cache_key() -> Dir:
+            remote_dir = await Dir.from_local("/tmp/data_dir/", dir_cache_key="my_cache_key_123")
+            return remote_dir
+        ```
         Args:
             local_path: Path to the local directory
-            remote_path: Optional path to store the directory remotely. If None, a path will be generated.
-            dir_cache_key: If you have a precomputed hash value you want to use when computing cache keys for
-              discoverable tasks that this File is an input to.
+            remote_destination: Optional remote path to store the directory. If None, a path will be automatically
+              generated.
+            dir_cache_key: Optional precomputed hash value to use for cache key computation when this Dir is used
+                          as an input to discoverable tasks. If not specified, the cache key will be based on
+                          directory attributes.
 
         Returns:
             A new Dir instance pointing to the uploaded directory
-
-        Example:
-        ```python
-        remote_dir = await Dir[DataFrame].from_local('/tmp/data_dir/', 's3://bucket/data/')
-        # With a known hash value you want to use for cache key calculation
-        remote_dir = await Dir[DataFrame].from_local('/tmp/data_dir/', 's3://bucket/data/', dir_cache_key='abc123')
-        ```
         """
         local_path_str = str(local_path)
         dirname = os.path.basename(os.path.normpath(local_path_str))
+        resolved_remote_path = remote_destination or internal_ctx().raw_data.get_random_remote_path(dirname)
+        protocol = get_protocol(resolved_remote_path)
+
+        # Shortcut for local, don't copy and just return
+        if "file" in protocol and remote_destination is None:
+            output_path = str(Path(local_path).absolute())
+            return cls(path=output_path, name=dirname, hash=dir_cache_key)
 
-        output_path = await storage.put(from_path=local_path_str, to_path=remote_path, recursive=True)
+        # todo: in the future, mirror File and set the file to_path here
+        output_path = await storage.put(from_path=local_path_str, to_path=remote_destination, recursive=True)
         return cls(path=output_path, name=dirname, hash=dir_cache_key)
 
     @classmethod
-    def from_existing_remote(cls, remote_path: str, dir_cache_key: Optional[str] = None) -> Dir[T]:
+    def from_local_sync(
+        cls,
+        local_path: Union[str, Path],
+        remote_destination: Optional[str] = None,
+        dir_cache_key: Optional[str] = None,
+    ) -> Dir[T]:
         """
-        Create a Dir reference from an existing remote directory.
+        Synchronously create a new Dir by uploading a local directory to remote storage.
 
-        Args:
-            remote_path: The remote path to the existing directory
-            dir_cache_key: Optional hash value to use for cache key computation. If not specified,
-                            the cache key will be computed based on this object's attributes.
+        Use this in non-async tasks when you have a local directory that needs to be uploaded to remote storage.
+
+        Example (Sync):
 
-        Example:
         ```python
-        remote_dir = Dir.from_existing_remote("s3://bucket/data/")
-        # With a known hash
-        remote_dir = Dir.from_existing_remote("s3://bucket/data/", dir_cache_key="abc123")
+        @env.task
+        def upload_local_directory_sync() -> Dir:
+            # Create a local directory with files
+            os.makedirs("/tmp/data_dir", exist_ok=True)
+            with open("/tmp/data_dir/file1.txt", "w") as f:
+                f.write("data1")
+
+            # Upload to remote storage
+            remote_dir = Dir.from_local_sync("/tmp/data_dir/")
+            return remote_dir
         ```
-        """
-        return cls(path=remote_path, hash=dir_cache_key)
 
-    @classmethod
-    def from_local_sync(cls, local_path: Union[str, Path], remote_path: Optional[str] = None) -> Dir[T]:
-        """
-        Synchronously create a new Dir by uploading a local directory to the configured remote store.
+        Example (Sync - With specific destination):
+
+        ```python
+        @env.task
+        def upload_to_specific_path_sync() -> Dir:
+            remote_dir = Dir.from_local_sync("/tmp/data_dir/", "s3://my-bucket/data/")
+            return remote_dir
+        ```
+
+        Example (Sync - With cache key):
+
+        ```python
+        @env.task
+        def upload_with_cache_key_sync() -> Dir:
+            remote_dir = Dir.from_local_sync("/tmp/data_dir/", dir_cache_key="my_cache_key_123")
+            return remote_dir
+        ```
 
         Args:
             local_path: Path to the local directory
-            remote_path: Optional path to store the directory remotely. If None, a path will be generated.
+            remote_destination: Optional remote path to store the directory. If None, a path will be automatically
+              generated.
+            dir_cache_key: Optional precomputed hash value to use for cache key computation when this Dir is used
+                          as an input to discoverable tasks. If not specified, the cache key will be based on
+                          directory attributes.
 
         Returns:
             A new Dir instance pointing to the uploaded directory
+        """
+        local_path_str = str(local_path)
+        dirname = os.path.basename(os.path.normpath(local_path_str))
+
+        resolved_remote_path = remote_destination or internal_ctx().raw_data.get_random_remote_path(dirname)
+        protocol = get_protocol(resolved_remote_path)
+
+        # Shortcut for local, don't copy and just return
+        if "file" in protocol and remote_destination is None:
+            output_path = str(Path(local_path).absolute())
+            return cls(path=output_path, name=dirname, hash=dir_cache_key)
+
+        fs = storage.get_underlying_filesystem(path=resolved_remote_path)
+        fs.put(local_path_str, resolved_remote_path, recursive=True)
+        return cls(path=resolved_remote_path, name=dirname, hash=dir_cache_key)
+
+    @classmethod
+    def from_existing_remote(cls, remote_path: str, dir_cache_key: Optional[str] = None) -> Dir[T]:
+        """
+        Create a Dir reference from an existing remote directory.
+
+        Use this when you want to reference a directory that already exists in remote storage without uploading it.
 
         Example:
+
         ```python
-        remote_dir = Dir[DataFrame].from_local_sync('/tmp/data_dir/', 's3://bucket/data/')
+        @env.task
+        async def process_existing_directory() -> int:
+            d = Dir.from_existing_remote("s3://my-bucket/data/")
+            files = await d.list_files()
+            return len(files)
         ```
+
+        Example (With cache key):
+
+        ```python
+        @env.task
+        async def process_with_cache_key() -> int:
+            d = Dir.from_existing_remote("s3://my-bucket/data/", dir_cache_key="abc123")
+            files = await d.list_files()
+            return len(files)
+        ```
+
+        Args:
+            remote_path: The remote path to the existing directory
+            dir_cache_key: Optional hash value to use for cache key computation. If not specified,
+                          the cache key will be computed based on the directory's attributes.
+
+        Returns:
+            A new Dir instance pointing to the existing remote directory
         """
-        # Implement this after we figure out the final sync story
-        raise NotImplementedError("Sync upload is not implemented for remote paths")
+        return cls(path=remote_path, hash=dir_cache_key)
 
     async def exists(self) -> bool:
         """
@@ -326,10 +729,15 @@ class Dir(BaseModel, Generic[T], SerializableType):
         Returns:
             True if the directory exists, False otherwise
 
-        Example:
+        Example (Async):
+
         ```python
-        if await directory.exists():
-            # Process the directory
+        @env.task
+        async def check_directory(d: Dir) -> bool:
+            if await d.exists():
+                print("Directory exists!")
+                return True
+            return False
         ```
         """
         fs = storage.get_underlying_filesystem(path=self.path)
@@ -342,13 +750,20 @@ class Dir(BaseModel, Generic[T], SerializableType):
         """
         Synchronously check if the directory exists.
 
+        Use this in non-async tasks or when you need synchronous directory existence checking.
+
         Returns:
             True if the directory exists, False otherwise
 
-        Example:
+        Example (Sync):
+
         ```python
-        if directory.exists_sync():
-            # Process the directory
+        @env.task
+        def check_directory_sync(d: Dir) -> bool:
+            if d.exists_sync():
+                print("Directory exists!")
+                return True
+            return False
         ```
         """
         fs = storage.get_underlying_filesystem(path=self.path)
@@ -356,20 +771,28 @@ class Dir(BaseModel, Generic[T], SerializableType):
 
     async def get_file(self, file_name: str) -> Optional[File[T]]:
         """
-        Asynchronously get a specific file from the directory.
+        Asynchronously get a specific file from the directory by name.
+
+        Use this when you know the name of a specific file in the directory you want to access.
+
+        Example (Async):
+
+        ```python
+        @env.task
+        async def read_specific_file(d: Dir) -> str:
+            file = await d.get_file("data.csv")
+            if file:
+                async with file.open("rb") as f:
+                    content = await f.read()
+                    return content.decode("utf-8")
+            return "File not found"
+        ```
 
         Args:
             file_name: The name of the file to get
 
         Returns:
             A File instance if the file exists, None otherwise
-
-        Example:
-        ```python
-        file = await directory.get_file("data.csv")
-        if file:
-            # Process the file
-        ```
         """
         fs = storage.get_underlying_filesystem(path=self.path)
         file_path = fs.sep.join([self.path, file_name])
@@ -381,20 +804,28 @@ class Dir(BaseModel, Generic[T], SerializableType):
 
     def get_file_sync(self, file_name: str) -> Optional[File[T]]:
         """
-        Synchronously get a specific file from the directory.
+        Synchronously get a specific file from the directory by name.
+
+        Use this in non-async tasks when you know the name of a specific file in the directory you want to access.
+
+        Example (Sync):
+
+        ```python
+        @env.task
+        def read_specific_file_sync(d: Dir) -> str:
+            file = d.get_file_sync("data.csv")
+            if file:
+                with file.open_sync("rb") as f:
+                    content = f.read()
+                    return content.decode("utf-8")
+            return "File not found"
+        ```
 
         Args:
             file_name: The name of the file to get
 
         Returns:
             A File instance if the file exists, None otherwise
-
-        Example:
-        ```python
-        file = directory.get_file_sync("data.csv")
-        if file:
-            # Process the file
-        ```
         """
         file_path = os.path.join(self.path, file_name)
         file = File[T](path=file_path)