ygg 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl

{ygg-0.1.24.dist-info → ygg-0.1.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.24
+Version: 0.1.26
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.24.dist-info → ygg-0.1.26.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ygg-0.1.24.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+ygg-0.1.26.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 yggdrasil/__init__.py,sha256=6OPibApplA5TF4TeixkQO_qewpaAidYX-fSDvvKYcTI,91
 yggdrasil/databricks/__init__.py,sha256=aGVve5mpoQtxSK2nfzrexjRPoutCIyaOnKZijkG4_QE,92
 yggdrasil/databricks/compute/__init__.py,sha256=TVDwPmW2SOmHmnhzZhsvrWbrxZ_lEcgqe3l9BeB-oxM,218
@@ -8,12 +8,12 @@ yggdrasil/databricks/compute/remote.py,sha256=sVWBb_1YR-e33on6F5QYMLKwT6end6rolC
 yggdrasil/databricks/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 yggdrasil/databricks/jobs/config.py,sha256=8Slfw4Wl7vu0kIlaUUqVqjjOgPwuULoo0rroENCbC20,11494
 yggdrasil/databricks/sql/__init__.py,sha256=JZpQ9eCphDf1l4yzIZ7a7OLigxqXkqOgb0Mio7Rj09A,181
-yggdrasil/databricks/sql/engine.py,sha256=WRDrY-FpXMegF3qNmMkCsfIitJB9rY_lW8Pnk7oo-xE,30837
+yggdrasil/databricks/sql/engine.py,sha256=bmkMoBij-8M-xohQK-sYb22OgBc1VIZRRVjw62CMWHk,30810
 yggdrasil/databricks/sql/exceptions.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 yggdrasil/databricks/sql/statement_result.py,sha256=L-hrK5MVnH3XG57BpGmaETtRzYjAtYGQthgAPGVj618,12610
 yggdrasil/databricks/sql/types.py,sha256=YgasSyq8sygk1h6ZOTcRwXAZWNKSuk-9g9VqlR8kJl4,5324
 yggdrasil/databricks/workspaces/__init__.py,sha256=tNNS3A_Pl9FYkQ8nGERhr4VF-hwKrvh8k1W8vTaR0uo,58
-yggdrasil/databricks/workspaces/databricks_path.py,sha256=ieaAEPo2lBmgqrdFwtXdy9uIDFyhFT8hjbO_KP5ayec,29547
+yggdrasil/databricks/workspaces/databricks_path.py,sha256=Ryoza649WRIHzbpIGjhvefwNFdSpe0-Y2ZJX6nImerw,31101
 yggdrasil/databricks/workspaces/workspace.py,sha256=8T-d0DTq-s2zTkmMbaEeS_7AiZwcUDxorch9IbZACko,33333
 yggdrasil/dataclasses/__init__.py,sha256=QVAvZnNl7gFYTLOVTfMkdQZf6o_WL8_UuDV1uTZ7Aeg,67
 yggdrasil/dataclasses/dataclass.py,sha256=ln-D1-bbiCLBd2khRMGs4dFoxzJEEGzHTKekWCnF2uk,5436
@@ -49,8 +49,8 @@ yggdrasil/types/cast/registry.py,sha256=-88mq-U1pDSGbEC9PRY0zJCzloyBodXgeSRBPb6h
 yggdrasil/types/cast/spark_cast.py,sha256=IHthM78dugabGXxNNW9sSHn-olDwzXcFdIFcPo9IiXU,23021
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=8PgJItF_XbyBcNuBnXkMQU3PBy3sAPEXZT9SXL2WbU4,4200
 yggdrasil/types/cast/spark_polars_cast.py,sha256=ba1UOvY1ouGCro1Np9slXmJ4TEyWnUtwVEAwxGvPLlk,8336
-ygg-0.1.24.dist-info/METADATA,sha256=w61_azInqpDmrls1mQUL-flgRhkNVmgLf1cbWJP0YUM,19204
-ygg-0.1.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.24.dist-info/entry_points.txt,sha256=D0JDw2s-ZfU1GjXfniv6PvXwcyn5v9WBk4ya623Cti8,71
-ygg-0.1.24.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.24.dist-info/RECORD,,
+ygg-0.1.26.dist-info/METADATA,sha256=3htWnURwJn36QKT6AYSn9syodfAo3ZdLa6qEN4cuUO8,19204
+ygg-0.1.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.26.dist-info/entry_points.txt,sha256=D0JDw2s-ZfU1GjXfniv6PvXwcyn5v9WBk4ya623Cti8,71
+ygg-0.1.26.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.26.dist-info/RECORD,,

yggdrasil/databricks/sql/engine.py

@@ -439,8 +439,7 @@ FROM parquet.`{databricks_tmp_folder}`"""
                 try:
                     databricks_tmp_folder.rmdir(recursive=True)
                 except Exception as e:
-                    raise e
-                    logger.error(e)
+                    logger.warning(e)
 
             # Optionally run OPTIMIZE / ZORDER / VACUUM if requested (Databricks SQL)
             if zorder_by:

yggdrasil/databricks/workspaces/databricks_path.py

@@ -7,7 +7,7 @@ import urllib.parse as urlparse
 from contextlib import contextmanager
 from enum import Enum
 from pathlib import PurePosixPath, Path as SysPath
-from typing import BinaryIO, Iterator, Optional, Tuple, Union, TYPE_CHECKING
+from typing import Any, BinaryIO, Iterator, Optional, Tuple, Union, TYPE_CHECKING
 
 from databricks.sdk.service.catalog import VolumeType
 
@@ -15,7 +15,14 @@ from ...libs.databrickslib import databricks
 
 if databricks is not None:
     from databricks.sdk.service.workspace import ImportFormat, ObjectType
-    from databricks.sdk.errors.platform import NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied, AlreadyExists, ResourceAlreadyExists
+    from databricks.sdk.errors.platform import (
+        NotFound,
+        ResourceDoesNotExist,
+        BadRequest,
+        PermissionDenied,
+        AlreadyExists,
+        ResourceAlreadyExists,
+    )
 
     NOT_FOUND_ERRORS = NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied
     ALREADY_EXISTS_ERRORS = AlreadyExists, ResourceAlreadyExists, BadRequest
@@ -26,7 +33,7 @@ if TYPE_CHECKING:
 
 __all__ = [
     "DatabricksPathKind",
-    "DatabricksPath"
+    "DatabricksPath",
 ]
 
 
@@ -43,7 +50,11 @@ class DatabricksPathKind(str, Enum):
     DBFS = "dbfs"
 
     @classmethod
-    def parse(cls, path: str, workspace: Optional["Workspace"] = None) -> Tuple["DatabricksPathKind", Optional["Workspace"], str]:
+    def parse(
+        cls,
+        path: str,
+        workspace: Optional["Workspace"] = None,
+    ) -> Tuple["DatabricksPathKind", Optional["Workspace"], str]:
         from .workspace import Workspace
 
         if path.startswith("/Workspace") or path.startswith("/Users") or path.startswith("/Shared"):
@@ -52,13 +63,19 @@ class DatabricksPathKind(str, Enum):
                 path = path.replace("/Users/me", "/Users/%s" % workspace.current_user.user_name)
 
             return cls.WORKSPACE, workspace, path
+
         if path.startswith("/Volumes"):
             return cls.VOLUME, workspace, path
 
         if path.startswith("dbfs://"):
             parsed = urlparse.urlparse(path)
-            kind, _, inner_path = cls.parse(parsed.path)
-            workspace = Workspace(host=parsed.hostname) if workspace is None else workspace
+
+            # inner path is the URL path (e.g. /tmp/x or /Volumes/...)
+            kind, _, inner_path = cls.parse(parsed.path, workspace=workspace)
+
+            # hostname can be None for malformed/dbfs:// variants; fall back to default Workspace()
+            if workspace is None:
+                workspace = Workspace(host=parsed.hostname) if parsed.hostname else Workspace()
 
             return kind, workspace, inner_path
 
@@ -66,7 +83,7 @@ class DatabricksPathKind(str, Enum):
 
 
 class DatabricksPath(SysPath, PurePosixPath):
-    _kind: DatabricksPathKind
+    _kind: "DatabricksPathKind"
     _workspace: Optional["Workspace"]
 
     _is_file: Optional[bool]
@@ -75,48 +92,121 @@ class DatabricksPath(SysPath, PurePosixPath):
     _raw_status: Optional[dict]
     _raw_status_refresh_time: float
 
+    @staticmethod
+    def _join_segments(pathsegments: tuple[Any, ...]) -> str:
+        if not pathsegments:
+            return ""
+
+        first = _seg_to_str(pathsegments[0])
+
+        # Keep dbfs:// URL-ish paths URL-ish (don't let PurePosixPath normalize it)
+        if first.startswith("dbfs://"):
+            rest = (_seg_to_str(s).lstrip("/") for s in pathsegments[1:])
+            first = first.rstrip("/")
+            tail = "/".join(rest)
+            return f"{first}/{tail}" if tail else first
+
+        return str(PurePosixPath(*(_seg_to_str(s) for s in pathsegments)))
+
+    def _init(self, template: Optional["DatabricksPath"] = None) -> None:
+        """
+        pathlib creates derived paths (parent, /, joinpath, with_name, etc.) via _from_parts
+        which bypasses __new__. _init(template=...) is the hook to carry our metadata forward.
+        """
+        if isinstance(template, DatabricksPath):
+            # Recompute kind for the NEW path string (don’t blindly copy _kind)
+            temp_ws = getattr(template, "_workspace", None)
+            kind, ws, _ = DatabricksPathKind.parse(str(self), workspace=temp_ws)
+
+            self._kind = kind
+            self._workspace = ws if ws is not None else temp_ws
+
+            # Never inherit caches from template
+            self._is_file = None
+            self._is_dir = None
+            self._raw_status = None
+            self._raw_status_refresh_time = 0.0
+        else:
+            kind, ws, _ = DatabricksPathKind.parse(str(self))
+            self._kind = kind
+            self._workspace = ws
+
+            self._is_file = None
+            self._is_dir = None
+            self._raw_status = None
+            self._raw_status_refresh_time = 0.0
+
     def __new__(
         cls,
-        *pathsegments,
+        *pathsegments: Any,
         workspace: Optional["Workspace"] = None,
         is_file: Optional[bool] = None,
         is_dir: Optional[bool] = None,
         raw_status: Optional[dict] = None,
-        raw_status_refresh_time: float = 0
-    ):
-        if not pathsegments:
-            joined = ""
-        else:
-            first = _seg_to_str(pathsegments[0])
-
-            # Special case: if someone passes a dbfs://... URL segment, keep it URL-like
-            if first.startswith("dbfs://"):
-                rest = [_seg_to_str(s).lstrip("/") for s in pathsegments[1:]]
-                joined = first.rstrip("/")
-                if rest:
-                    joined += "/" + "/".join(rest)
-            else:
-                joined = str(PurePosixPath(*(_seg_to_str(s) for s in pathsegments)))
+        raw_status_refresh_time: float = 0.0,
+    ) -> "DatabricksPath":
+        joined = cls._join_segments(pathsegments)
+        kind, parsed_ws, pure_path = DatabricksPathKind.parse(joined, workspace=workspace)
+
+        self = cls._from_parts([pure_path])  # pathlib-style construction (calls _init)
+
+        # Override with constructor-provided metadata
+        self._kind = kind
+        self._workspace = parsed_ws if workspace is None else workspace
+        self._is_file = is_file
+        self._is_dir = is_dir
+        self._raw_status = raw_status
+        self._raw_status_refresh_time = float(raw_status_refresh_time)
+
+        return self
 
-        kind, w, p = DatabricksPathKind.parse(joined)
+    def __init__(
+        self,
+        *pathsegments: Any,
+        workspace: Optional["Workspace"] = None,
+        is_file: Optional[bool] = None,
+        is_dir: Optional[bool] = None,
+        raw_status: Optional[dict] = None,
+        raw_status_refresh_time: float = 0.0,
+    ) -> None:
+        # pathlib paths are effectively immutable; all init happens in __new__ / _init
+        pass
 
-        obj = super().__new__(cls, p)
+    def __truediv__(self, other):
+        if not other:
+            return self
 
-        obj._kind = kind
-        obj._workspace = w if workspace is None else workspace
-        obj._is_file = is_file
-        obj._is_dir = is_dir
-        obj._raw_status = raw_status
-        obj._raw_status_refresh_time = raw_status_refresh_time
+        built = super().__truediv__(other)
 
-        return obj
+        built._kind = self._kind
+        built._workspace = self._workspace
+
+        built._is_file = None
+        built._is_dir = None
+        built._raw_status = None
+        built._raw_status_refresh_time = 0.0
+
+        return built
 
     def __enter__(self):
         self.workspace.__enter__()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.workspace.__exit__(exc_type, exc_val, exc_tb)
+        return self.workspace.__exit__(exc_type, exc_val, exc_tb)
+
+    def _from_parsed_parts(self, drv, root, parts):
+        built = super()._from_parsed_parts(drv, root, parts)
+
+        built._kind = self._kind
+        built._workspace = self._workspace
+
+        built._is_file = None
+        built._is_dir = None
+        built._raw_status = None
+        built._raw_status_refresh_time = 0.0
+
+        return built
 
     @property
     def workspace(self):
@@ -126,22 +216,30 @@ class DatabricksPath(SysPath, PurePosixPath):
             self._workspace = Workspace()
         return self._workspace
 
+    @workspace.setter
+    def workspace(self, value):
+        self._workspace = value
+
     @property
     def kind(self):
         return self._kind
 
-    def is_file(self, *, follow_symlinks = True):
+    @kind.setter
+    def kind(self, value: DatabricksPathKind):
+        self._kind = value
+
+    def is_file(self, *, follow_symlinks=True):
         if self._is_file is None:
             self.refresh_status()
         return self._is_file
 
-    def is_dir(self, *, follow_symlinks = True):
+    def is_dir(self, *, follow_symlinks=True):
         if self._is_dir is None:
             self.refresh_status()
         return self._is_dir
 
     def volume_parts(self) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[PurePosixPath]]:
-        if self._kind != DatabricksPathKind.VOLUME:
+        if self.kind != DatabricksPathKind.VOLUME:
             return None, None, None, None
 
         s = str(self)
@@ -169,12 +267,12 @@ class DatabricksPath(SysPath, PurePosixPath):
             sdk = connected.workspace.sdk()
 
             try:
-                if connected._kind == DatabricksPathKind.VOLUME:
+                if connected.kind == DatabricksPathKind.VOLUME:
                     info = sdk.files.get_metadata(connected.as_files_api_path())
 
                     connected._raw_status = info
                     connected._is_file, connected._is_dir = True, False
-                elif connected._kind == DatabricksPathKind.WORKSPACE:
+                elif connected.kind == DatabricksPathKind.WORKSPACE:
                     info = sdk.workspace.get_status(connected.as_workspace_api_path())
 
                     is_dir = info.object_type in (ObjectType.DIRECTORY, ObjectType.REPO)
@@ -184,7 +282,7 @@ class DatabricksPath(SysPath, PurePosixPath):
                     info = sdk.dbfs.get_status(connected.as_dbfs_api_path())
 
                     connected._raw_status = info
-                    connected._is_file, connected._is_dir = not info.is_dir, info.is_dir
+                    connected._is_file, connected._is_dir = (not info.is_dir), info.is_dir
 
                 connected._raw_status_refresh_time = time.time()
             except (NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied):
@@ -204,7 +302,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         self._is_file = None
         self._is_dir = None
 
-
     # ---- API path normalization helpers ----
 
     def as_workspace_api_path(self) -> str:
@@ -243,7 +340,7 @@ class DatabricksPath(SysPath, PurePosixPath):
             return True
         return False
 
-    def mkdir(self, mode = 0o777, parents = True, exist_ok = True):
+    def mkdir(self, mode=0o777, parents=True, exist_ok=True):
         """
         Create a new directory at this given path.
         """
@@ -251,9 +348,9 @@ class DatabricksPath(SysPath, PurePosixPath):
             connected.clear_cache()
 
             try:
-                if connected._kind == DatabricksPathKind.WORKSPACE:
+                if connected.kind == DatabricksPathKind.WORKSPACE:
                     connected.workspace.sdk().workspace.mkdirs(self.as_workspace_api_path())
-                elif connected._kind == DatabricksPathKind.VOLUME:
+                elif connected.kind == DatabricksPathKind.VOLUME:
                     return connected._create_volume_dir(mode=mode, parents=parents, exist_ok=exist_ok)
                 elif connected._kind == DatabricksPathKind.DBFS:
                     connected.workspace.sdk().dbfs.mkdirs(self.as_dbfs_api_path())
@@ -266,8 +363,6 @@ class DatabricksPath(SysPath, PurePosixPath):
                 connected.parent.mkdir(parents=True, exist_ok=True)
                 connected.mkdir(mode, parents=False, exist_ok=exist_ok)
             except (AlreadyExists, ResourceAlreadyExists):
-                # Cannot rely on checking for EEXIST, since the operating system
-                # could give priority to other errors like EACCES or EROFS
                 if not exist_ok:
                     raise
 
@@ -279,8 +374,6 @@ class DatabricksPath(SysPath, PurePosixPath):
             try:
                 sdk.catalogs.create(name=catalog_name)
             except (AlreadyExists, ResourceAlreadyExists, PermissionDenied, BadRequest):
-                # Cannot rely on checking for EEXIST, since the operating system
-                # could give priority to other errors like EACCES or EROFS
                 if not exist_ok:
                     raise
 
@@ -288,8 +381,6 @@ class DatabricksPath(SysPath, PurePosixPath):
             try:
                 sdk.schemas.create(catalog_name=catalog_name, name=schema_name)
             except (AlreadyExists, ResourceAlreadyExists, PermissionDenied, BadRequest):
-                # Cannot rely on checking for EEXIST, since the operating system
-                # could give priority to other errors like EACCES or EROFS
                 if not exist_ok:
                     raise
 
@@ -299,15 +390,13 @@ class DatabricksPath(SysPath, PurePosixPath):
                     catalog_name=catalog_name,
                     schema_name=schema_name,
                     name=volume_name,
-                    volume_type=VolumeType.MANAGED
+                    volume_type=VolumeType.MANAGED,
                 )
             except (AlreadyExists, ResourceAlreadyExists, BadRequest):
-                # Cannot rely on checking for EEXIST, since the operating system
-                # could give priority to other errors like EACCES or EROFS
                 if not exist_ok:
                     raise
 
-    def _create_volume_dir(self, mode = 0o777, parents = True, exist_ok = True):
+    def _create_volume_dir(self, mode=0o777, parents=True, exist_ok=True):
         path = self.as_files_api_path()
         sdk = self.workspace.sdk()
 
@@ -324,8 +413,6 @@ class DatabricksPath(SysPath, PurePosixPath):
 
             sdk.files.create_directory(path)
         except (AlreadyExists, ResourceAlreadyExists, BadRequest):
-            # Cannot rely on checking for EEXIST, since the operating system
-            # could give priority to other errors like EACCES or EROFS
             if not exist_ok:
                 raise
 
@@ -340,11 +427,11 @@ class DatabricksPath(SysPath, PurePosixPath):
 
     def rmfile(self):
         try:
-            if self._kind == DatabricksPathKind.VOLUME:
+            if self.kind == DatabricksPathKind.VOLUME:
                 return self._remove_volume_file()
-            elif self._kind == DatabricksPathKind.WORKSPACE:
+            elif self.kind == DatabricksPathKind.WORKSPACE:
                 return self._remove_workspace_file()
-            elif self._kind == DatabricksPathKind.DBFS:
+            elif self.kind == DatabricksPathKind.DBFS:
                 return self._remove_dbfs_file()
         finally:
             self.clear_cache()
@@ -376,17 +463,17 @@ class DatabricksPath(SysPath, PurePosixPath):
     def rmdir(self, recursive: bool = True):
         with self as connected:
             try:
-                if connected._kind == DatabricksPathKind.WORKSPACE:
+                if connected.kind == DatabricksPathKind.WORKSPACE:
                     connected.workspace.sdk().workspace.delete(
                         self.as_workspace_api_path(),
-                        recursive=recursive
+                        recursive=recursive,
                     )
-                elif connected._kind == DatabricksPathKind.VOLUME:
+                elif connected.kind == DatabricksPathKind.VOLUME:
                     return self._remove_volume_dir(recursive=recursive)
                 else:
                     connected.workspace.sdk().dbfs.delete(
                         self.as_dbfs_api_path(),
-                        recursive=recursive
+                        recursive=recursive,
                     )
             except (NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied):
                 pass
@@ -425,13 +512,13 @@ class DatabricksPath(SysPath, PurePosixPath):
         self.clear_cache()
 
     def ls(self, recursive: bool = False, fetch_size: int = None, raise_error: bool = True):
-        if self._kind == DatabricksPathKind.VOLUME:
+        if self.kind == DatabricksPathKind.VOLUME:
             for _ in self._ls_volume(recursive=recursive, fetch_size=fetch_size, raise_error=raise_error):
                 yield _
-        elif self._kind == DatabricksPathKind.WORKSPACE:
+        elif self.kind == DatabricksPathKind.WORKSPACE:
             for _ in self._ls_workspace(recursive=recursive, fetch_size=fetch_size, raise_error=raise_error):
                 yield _
-        elif self._kind == DatabricksPathKind.DBFS:
+        elif self.kind == DatabricksPathKind.DBFS:
             for _ in self._ls_dbfs(recursive=recursive, fetch_size=fetch_size, raise_error=raise_error):
                 yield _
 
@@ -444,13 +531,13 @@ class DatabricksPath(SysPath, PurePosixPath):
                 try:
                     for info in sdk.volumes.list(
                         catalog_name=catalog_name,
-                        schema_name=schema_name
+                        schema_name=schema_name,
                     ):
                         base = DatabricksPath(
                             f"/Volumes/{info.catalog_name}/{info.schema_name}/{info.name}",
                             workspace=self.workspace,
                             is_file=False,
-                            is_dir=True
+                            is_dir=True,
                         )
 
                         if recursive:
@@ -463,12 +550,12 @@ class DatabricksPath(SysPath, PurePosixPath):
                         raise
             elif schema_name is None:
                 try:
-                    for info in sdk.schemas.list(catalog_name=catalog_name,):
+                    for info in sdk.schemas.list(catalog_name=catalog_name):
                         base = DatabricksPath(
                             f"/Volumes/{info.catalog_name}/{info.name}",
                             workspace=self.workspace,
                             is_file=False,
-                            is_dir=True
+                            is_dir=True,
                         )
 
                         if recursive:
@@ -486,7 +573,7 @@ class DatabricksPath(SysPath, PurePosixPath):
                             f"/Volumes/{info.name}",
                             workspace=self.workspace,
                             is_file=False,
-                            is_dir=True
+                            is_dir=True,
                         )
 
                         if recursive:
@@ -504,7 +591,7 @@ class DatabricksPath(SysPath, PurePosixPath):
                         info.path,
                         workspace=self.workspace,
                         is_file=not info.is_directory,
-                        is_dir=info.is_directory
+                        is_dir=info.is_directory,
                     )
 
                     if recursive and info.is_directory:
@@ -526,9 +613,8 @@ class DatabricksPath(SysPath, PurePosixPath):
                     info.path,
                     workspace=self.workspace,
                     is_file=not is_dir,
-                    is_dir=is_dir
+                    is_dir=is_dir,
                 )
-
                 yield base
         except (NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied):
             if raise_error:
@@ -538,14 +624,15 @@ class DatabricksPath(SysPath, PurePosixPath):
         sdk = self.workspace.sdk()
 
         try:
-            for info in sdk.dbfs.list(self.as_workspace_api_path(), recursive=recursive):
+            # FIX: DBFS listing should use DBFS-normalized path, not workspace path
+            p = "/dbfs/" + self.as_dbfs_api_path() + "/"
+            for info in sdk.dbfs.list(p, recursive=recursive):
                 base = DatabricksPath(
                     info.path,
                     workspace=self.workspace,
                     is_file=not info.is_dir,
-                    is_dir=info.is_dir
+                    is_dir=info.is_dir,
                 )
-
                 yield base
         except (NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied):
             if raise_error:
@@ -554,7 +641,7 @@ class DatabricksPath(SysPath, PurePosixPath):
     @contextmanager
     def open(
         self,
-        mode='r',
+        mode="r",
         buffering=-1,
         encoding=None,
         errors=None,
@@ -569,11 +656,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         Supported:
           - read:  "rb", "r"
           - write: "wb", "w"  (buffered; uploads on close for WORKSPACE/VOLUME)
-
-        Notes:
-          - VOLUME: uses w.files.download/upload (Files API). :contentReference[oaicite:5]{index=5}
-          - DBFS: uses w.dbfs.open when possible. :contentReference[oaicite:6]{index=6}
-          - WORKSPACE: uses w.workspace.download/upload. :contentReference[oaicite:7]{index=7}
         """
         if mode not in {"rb", "r", "wb", "w"}:
             raise ValueError(f"Unsupported mode {mode!r}. Use r/rb/w/wb.")
@@ -592,10 +674,10 @@ class DatabricksPath(SysPath, PurePosixPath):
     @contextmanager
     def open_read(self, encoding: str | None = None):
         with self as connected:
-            if connected._kind == DatabricksPathKind.VOLUME:
+            if connected.kind == DatabricksPathKind.VOLUME:
                 with connected._open_read_volume(encoding=encoding) as f:
                     yield f
-            elif connected._kind == DatabricksPathKind.WORKSPACE:
+            elif connected.kind == DatabricksPathKind.WORKSPACE:
                 with connected._open_read_workspace(encoding=encoding) as f:
                     yield f
             else:
@@ -607,7 +689,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         workspace_client = self.workspace.sdk()
         path = self.as_files_api_path()
 
-        # Files.download returns a stream-like response body. :contentReference[oaicite:8]{index=8}
         resp = workspace_client.files.download(path)
         raw = io.BytesIO(resp.contents.read())
 
@@ -623,12 +704,10 @@ class DatabricksPath(SysPath, PurePosixPath):
         workspace_client = self.workspace.sdk()
         path = self.as_workspace_api_path()
 
-        # Files.download returns a stream-like response body. :contentReference[oaicite:8]{index=8}
-        raw = workspace_client.workspace.download(path)  # returns BinaryIO :contentReference[oaicite:10]{index=10}
+        raw = workspace_client.workspace.download(path)  # returns BinaryIO
 
         if encoding is not None:
             raw = io.BytesIO(raw.read())
-
             with io.TextIOWrapper(raw, encoding=encoding) as f:
                 yield f
         else:
@@ -640,7 +719,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         workspace_client = self.workspace.sdk()
         path = self.as_dbfs_api_path()
 
-        # dbfs.open gives BinaryIO for streaming reads :contentReference[oaicite:12]{index=12}
         raw = workspace_client.dbfs.open(path, read=True)
 
         if encoding is not None:
@@ -653,10 +731,10 @@ class DatabricksPath(SysPath, PurePosixPath):
     @contextmanager
     def open_write(self, encoding: str | None = None):
         with self as connected:
-            if connected._kind == DatabricksPathKind.VOLUME:
+            if connected.kind == DatabricksPathKind.VOLUME:
                 with connected._open_write_volume(encoding=encoding) as f:
                     yield f
-            elif connected._kind == DatabricksPathKind.WORKSPACE:
+            elif connected.kind == DatabricksPathKind.WORKSPACE:
                 with connected._open_write_workspace(encoding=encoding) as f:
                     yield f
             else:
@@ -668,7 +746,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         workspace_client = self.workspace.sdk()
         path = self.as_files_api_path()
 
-        # Buffer locally then upload stream on exit. :contentReference[oaicite:9]{index=9}
         buf = io.BytesIO()
 
         if encoding is not None:
@@ -703,7 +780,6 @@ class DatabricksPath(SysPath, PurePosixPath):
         workspace_client = self.workspace.sdk()
         path = self.as_workspace_api_path()
 
-        # Buffer then upload (AUTO works for workspace files) :contentReference[oaicite:11]{index=11}
         buf = io.BytesIO()
 
         if encoding is not None:
@@ -727,7 +803,7 @@ class DatabricksPath(SysPath, PurePosixPath):
                             path, buf, format=ImportFormat.AUTO, overwrite=overwrite
                         )
                     else:
-                        raise e
+                        raise
 
                 tw.detach()
         else:
@@ -749,14 +825,14 @@ class DatabricksPath(SysPath, PurePosixPath):
                             path, buf, format=ImportFormat.AUTO, overwrite=overwrite
                         )
                     else:
-                        raise e
+                        raise
 
     @contextmanager
     def _open_write_dbfs(self, encoding: str | None = None, overwrite: bool = True):
         workspace_client = self.workspace.sdk()
         path = self.as_dbfs_api_path()
 
-        raw = workspace_client.dbfs.open(path, write=True, overwrite=overwrite)  # :contentReference[oaicite:13]{index=13}
+        raw = workspace_client.dbfs.open(path, write=True, overwrite=overwrite)
 
         if encoding is not None:
             with io.TextIOWrapper(raw, encoding=encoding) as f:
@@ -766,4 +842,4 @@ class DatabricksPath(SysPath, PurePosixPath):
                 yield f
 
         self.clear_cache()
-        self._is_file, self._is_dir = True, False
+        self._is_file, self._is_dir = True, False