fabricks 3.0.19__py3-none-any.whl → 4.0.0__py3-none-any.whl

fabricks/utils/path.py

@@ -1,23 +1,25 @@
-import os
 import posixpath
+from abc import ABC, abstractmethod
 from pathlib import Path as PathlibPath
 from typing import List, Optional, Union
 
 from pyspark.sql.dataframe import DataFrame
+from typing_extensions import deprecated
 
 from fabricks.utils.spark import spark
 
 
-class Path:
-    def __init__(self, path: Union[str, PathlibPath], assume_git: bool = False):
-        self.assume_git = assume_git
+class BasePath(ABC):
+    """Abstract base class for all path types."""
 
+    def __init__(self, path: Union[str, PathlibPath]):
+        """Initialize the path."""
         if isinstance(path, PathlibPath):
             path = path.as_posix()
 
         new_path = str(path)
         if new_path.startswith("abfss:/") and not new_path.startswith("abfss://"):
-            new_path = new_path.replace("abfss:/", "abfss://")  # // is replaced by / by pathlibpath
+            new_path = new_path.replace("abfss:/", "abfss://")
 
         self.path: str = new_path
 
@@ -26,64 +28,100 @@ class Path:
         cls,
         uri: str,
         regex: Optional[dict[str, str]] = None,
-        assume_git: Optional[bool] = False,
     ):
+        """Create a path from a URI with optional regex substitution."""
         uri = uri.strip()
-        if assume_git is None:
-            assume_git = False
         if regex:
             import re
 
             for key, value in regex.items():
                 uri = re.sub(rf"{key}", value, uri)
 
-        return cls(uri, assume_git=assume_git)
+        return cls(uri)
+
+    @property
+    def string(self) -> str:
+        """Get the string representation of the path."""
+        return self.path
 
     @property
     def pathlibpath(self) -> PathlibPath:
+        """Get the pathlib representation of the path."""
         return PathlibPath(self.string)
 
-    @property
-    def string(self) -> str:
-        return self.path
+    def get_file_name(self) -> str:
+        """Get the file name from the path."""
+        return self.pathlibpath.name
 
-    def get_container(self) -> str:
-        import re
+    def get_sql(self) -> str:
+        """Read and return SQL content from a .sql file."""
+        p = self.string
+        if not p.endswith(".sql"):
+            p += ".sql"
 
-        assert self.string.startswith("abfss://")
+        with open(p, "r") as f:
+            sql = f.read()
 
-        r = re.compile(r"(?<=abfss:\/\/)(.+?)(?=@)")
-        m = re.findall(r, self.string)[0]
-        return m
+        return sql
 
-    def get_storage_account(self) -> str:
-        import re
+    def is_sql(self) -> bool:
+        """Check if the path points to a SQL file."""
+        return self.string.endswith(".sql")
 
-        assert self.string.startswith("abfss://")
+    def joinpath(self, *other):
+        """Join this path with other path segments."""
+        parts = [str(o) for o in other]
+        base = self.string
 
-        r = re.compile(r"(?<=@)(.+?)(?=\.)")
-        m = re.findall(r, self.string)[0]
-        return m
+        joined = posixpath.join(base, *parts)
+        new = posixpath.normpath(joined)
 
-    def get_file_name(self) -> str:
-        return self.pathlibpath.name
+        return self.__class__(path=new)
 
-    def get_file_system(self) -> str:
-        import re
+    def append(self, other: str):
+        """Append a string to the path."""
+        new_path = self.string + other
+        return self.__class__(path=new_path)
 
-        assert self.string.startswith("abfss://")
+    def parent(self):
+        """Get the parent directory of the path."""
+        new_path = self.pathlibpath.parent
+        return self.__class__(path=new_path)
 
-        r = re.compile(r"(?<=\.)(.+)(?=\/)")
-        m = re.findall(r, self.string)[0]
-        return m
+    @abstractmethod
+    def exists(self) -> bool:
+        """Check if the path exists."""
 
-    def get_dbfs_mnt_path(self) -> str:
-        mount_point = self.pathlibpath.parts[1].split(".")[0].split("@")[0]
-        rest = self.pathlibpath.parts[2:]
+    @abstractmethod
+    def walk(
+        self,
+        depth: Optional[int] = None,
+        convert: Optional[bool] = False,
+        file_format: Optional[str] = None,
+    ) -> List:
+        """Walk the path and return all files."""
 
-        return str(os.path.join("/dbfs/mnt", mount_point, "/".join(rest)))
+    @abstractmethod
+    def _yield(self, path: Union[str, PathlibPath]):
+        """Recursively yield all file paths under the given path."""
+
+    def __str__(self) -> str:
+        return self.string
+
+
+class GitPath(BasePath):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path)
+
+    def exists(self) -> bool:
+        """Check if the path exists in the local/git file system."""
+        try:
+            return self.pathlibpath.exists()
+        except Exception:
+            return False
 
     def get_notebook_path(self) -> str:
+        """Get the notebook path for Databricks workspace."""
         path = self.path.replace("Workspace/", "")
         if path.endswith(".ipynb"):
             path = path.replace(".ipynb", "")
@@ -91,61 +129,90 @@ class Path:
             path = path.replace(".py", "")
         return path
 
-    def get_sql(self) -> str:
-        p = self.string
-        if not p.endswith(".sql"):
-            p += ".sql"
+    def walk(
+        self,
+        depth: Optional[int] = None,
+        convert: Optional[bool] = False,
+        file_format: Optional[str] = None,
+    ) -> List:
+        out = []
+        if self.exists():
+            if self.pathlibpath.is_file():
+                out = [self.string]
+            else:
+                out = list(self._yield(self.string))
 
-        with open(p, "r") as f:
-            sql = f.read()
+        if file_format:
+            out = [o for o in out if o.endswith(".sql")]
+        if convert:
+            out = [self.__class__(o) for o in out]
+        return out
 
-        return sql
+    def _yield(self, path: Union[str, PathlibPath]):
+        """Recursively yield all file paths in the git/local file system."""
+        if isinstance(path, str):
+            path = PathlibPath(path)
 
-    def is_sql(self) -> bool:
-        return self.string.endswith(".sql")
+        for child in path.glob(r"*"):
+            if child.is_dir():
+                yield from self._yield(child)
+            else:
+                yield str(child)
+
+
+class FileSharePath(BasePath):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path)
 
     def exists(self) -> bool:
+        """Check if the path exists in the distributed file system."""
         try:
-            if self.assume_git:
-                return self.pathlibpath.exists()
+            from fabricks.utils.spark import dbutils
 
-            else:
-                from fabricks.utils.spark import dbutils
+            assert dbutils is not None, "dbutils not found"
+            dbutils.fs.ls(self.string)
+            return True
+        except Exception:
+            return False
 
-                assert dbutils is not None, "dbutils not found"
+    def get_container(self) -> str:
+        """Get the container name from an ABFSS path."""
+        import re
 
-                dbutils.fs.ls(self.string)
-                return True
+        assert self.string.startswith("abfss://")
 
-        except Exception:
-            return False
+        r = re.compile(r"(?<=abfss:\/\/)(.+?)(?=@)")
+        m = re.findall(r, self.string)[0]
+        return m
 
-    def joinpath(self, *other):
-        parts = [str(o) for o in other]
-        base = self.string
+    def get_storage_account(self) -> str:
+        """Get the storage account name from an ABFSS path."""
+        import re
 
-        joined = posixpath.join(base, *parts)
-        new = posixpath.normpath(joined)
+        assert self.string.startswith("abfss://")
 
-        return Path(path=new, assume_git=self.assume_git)
+        r = re.compile(r"(?<=@)(.+?)(?=\.)")
+        m = re.findall(r, self.string)[0]
+        return m
 
-    def append(self, other: str):
-        new_path = self.string + other
-        return Path(path=new_path, assume_git=self.assume_git)
+    def get_file_system(self) -> str:
+        """Get the file system from an ABFSS path."""
+        import re
 
-    def parent(self, *other):
-        new_path = self.pathlibpath.parent
-        return Path(path=new_path, assume_git=self.assume_git)
+        assert self.string.startswith("abfss://")
 
-    def get_file_info(self) -> DataFrame:
-        assert not self.assume_git
+        r = re.compile(r"(?<=\.)(.+)(?=\/)")
+        m = re.findall(r, self.string)[0]
+        return m
 
-        rows = self._yield_file_info(self.string)
-        df = spark.createDataFrame(
-            rows,
-            schema=["path", "name", "size", "modification_time"],
-        )
-        return df
+    def get_dbfs_mnt_path(self) -> str:
+        """Get the DBFS mount path."""
+        import os
+
+        mount_point = self.pathlibpath.parts[1].split(".")[0].split("@")[0]
+        rest = self.pathlibpath.parts[2:]
+
+        return str(os.path.join("/dbfs/mnt", mount_point, "/".join(rest)))
 
     def walk(
         self,
@@ -158,20 +225,31 @@ class Path:
             if self.pathlibpath.is_file():
                 out = [self.string]
             elif depth:
-                assert not self.assume_git
                 out = self._list_fs(depth)
             else:
-                if self.assume_git:
-                    out = list(self._yield_git(self.string))
-                else:
-                    out = list(self._yield_fs(self.string))
+                out = list(self._yield(self.string))
 
         if file_format:
             out = [o for o in out if o.endswith(".sql")]
         if convert:
-            out = [Path(o) for o in out]
+            out = [self.__class__(o) for o in out]
         return out
 
+    def get_file_info(self) -> DataFrame:
+        rows = self._yield_file_info(self.string)
+        df = spark.createDataFrame(
+            rows,
+            schema=["path", "name", "size", "modification_time"],
+        )
+        return df
+
+    def rm(self):
+        from databricks.sdk.runtime import dbutils
+
+        if self.exists():
+            list(self._rm(self.string))
+            dbutils.fs.rm(self.string, recurse=True)
+
     def _list_fs(self, depth: int) -> List:
         from databricks.sdk.runtime import dbutils
 
@@ -205,32 +283,17 @@ class Path:
             else:
                 yield dbutils.fs.ls(child.path)[0]
 
-    def _yield_fs(self, path: str):
+    def _yield(self, path: Union[str, PathlibPath]):
+        """Recursively yield all file paths in the distributed file system."""
         from databricks.sdk.runtime import dbutils
 
-        for child in dbutils.fs.ls(path):
+        path_str = str(path)
+        for child in dbutils.fs.ls(path_str):
             if child.isDir():  # type: ignore
-                yield from self._yield_fs(child.path)
+                yield from self._yield(child.path)
             else:
                 yield str(child.path)
 
-    def _yield_git(self, path: Union[str, PathlibPath]):
-        if isinstance(path, str):
-            path = PathlibPath(path)
-
-        for child in path.glob(r"*"):
-            if child.is_dir():
-                yield from self._yield_git(child)
-            else:
-                yield str(child)
-
-    def rm(self):
-        from databricks.sdk.runtime import dbutils
-
-        if self.exists():
-            list(self._rm(self.string))
-            dbutils.fs.rm(self.string, recurse=True)
-
     def _rm(self, path: str):
         from databricks.sdk.runtime import dbutils
 
@@ -244,15 +307,109 @@ class Path:
         except Exception:
             return False
 
-    def __str__(self) -> str:
-        return self.string
 
+def resolve_git_path(
+    path: str | None,
+    default: str | None = None,
+    base: GitPath | str | None = None,
+    variables: dict[str, str] | None = None,
+) -> GitPath:
+    """
+    Resolve a path as a GitPath with optional variable substitution and base path joining.
 
-class GitPath(Path):
-    def __init__(self, path: Union[str, PathlibPath]):
-        super().__init__(path=path, assume_git=True)
+    Args:
+        path: The path string from configuration
+        default: Default value if path is None
+        base: Base path to join with (must be GitPath or str)
+        apply_variables: Whether to apply variable substitution using VARIABLES
+        variables: Dictionary of variable substitutions
 
+    Returns:
+        Resolved GitPath object
+    """
+    if isinstance(base, str):
+        base = GitPath(base)
 
-class FileSharePath(Path):
-    def __init__(self, path: Union[str, PathlibPath]):
-        super().__init__(path=path, assume_git=False)
+    resolved_value = path or default
+    if resolved_value is None:
+        raise ValueError("path and default cannot both be None")
+
+    if variables:
+        return GitPath.from_uri(resolved_value, regex=variables)
+
+    if base:
+        return base.joinpath(resolved_value)
+
+    return GitPath(resolved_value)
+
+
+def resolve_fileshare_path(
+    path: str | None,
+    default: str | None = None,
+    base: FileSharePath | str | None = None,
+    variables: dict[str, str] | None = None,
+) -> FileSharePath:
+    """
+    Resolve a path as a FileSharePath with optional variable substitution and base path joining.
+
+    Args:
+        path: The path string from configuration
+        default: Default value if path is None
+        base: Base path to join with (must be FileSharePath or str)
+        apply_variables: Whether to apply variable substitution using VARIABLES
+        variables: Dictionary of variable substitutions
+
+    Returns:
+        Resolved FileSharePath object
+    """
+    if isinstance(base, str):
+        base = FileSharePath(base)
+
+    resolved_value = path or default
+    if resolved_value is None:
+        raise ValueError("path and default cannot both be None")
+
+    if variables:
+        return FileSharePath.from_uri(resolved_value, regex=variables)
+
+    if base:
+        return base.joinpath(resolved_value)
+
+    return FileSharePath(resolved_value)
+
+
+@deprecated("Use GitPath or FileSharePath directly instead.")
+class Path:
+    """
+    Legacy Path class with assume_git flag for backward compatibility.
+    """
+
+    def __new__(cls, path: Union[str, PathlibPath], assume_git: bool = False):
+        if assume_git:
+            return GitPath(path)
+        else:
+            return FileSharePath(path)
+
+    @classmethod
+    def from_uri(
+        cls,
+        uri: str,
+        regex: Optional[dict[str, str]] = None,
+        assume_git: Optional[bool] = False,
+    ):
+        """
+        Create a path from a URI with optional regex substitution.
+
+        Args:
+            uri: The URI string
+            regex: Dictionary of regex patterns to substitute
+            assume_git: If True, return GitPath; otherwise FileSharePath
+
+        Returns:
+            GitPath if assume_git is True, FileSharePath otherwise
+        """
+        if assume_git is None:
+            assume_git = False
+
+        path_class = GitPath if assume_git else FileSharePath
+        return path_class.from_uri(uri, regex=regex)

fabricks/utils/pip.py

@@ -1,13 +1,13 @@
 import subprocess
 from typing import List, Optional, Union
 
-from fabricks.utils.path import Path
+from fabricks.utils.path import FileSharePath
 
 
 def pip_package(
     package: Union[str, List[str]],
-    whl_path: Optional[Path] = None,
-    tgt_path: Optional[Path] = None,
+    whl_path: Optional[FileSharePath] = None,
+    tgt_path: Optional[FileSharePath] = None,
 ):
     if isinstance(package, str):
         package = [package]
@@ -29,9 +29,9 @@ def pip_package(
 
 
 def pip_requirements(
-    requirements_path: Path,
-    whl_path: Optional[Path] = None,
-    tgt_path: Optional[Path] = None,
+    requirements_path: FileSharePath,
+    whl_path: Optional[FileSharePath] = None,
+    tgt_path: Optional[FileSharePath] = None,
 ):
     r = requirements_path.string
 
@@ -50,7 +50,7 @@ def pip_requirements(
         raise ValueError(r, out.stderr)
 
 
-def pip_wheel(requirement_path: Path, whl_path: Path):
+def pip_wheel(requirement_path: FileSharePath, whl_path: FileSharePath):
     import subprocess
 
     r = requirement_path.string

fabricks/utils/read/read.py

@@ -4,12 +4,12 @@ from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import StructType
 
 from fabricks.context import SPARK
-from fabricks.utils.path import Path
+from fabricks.utils.path import FileSharePath
 
 
 @overload
 def read_stream(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     *,
     schema: StructType,
@@ -20,9 +20,9 @@ def read_stream(
 
 @overload
 def read_stream(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
-    schema_path: Union[Path, str],
+    schema_path: Union[FileSharePath, str],
     *,
     options: Optional[dict[str, str]] = None,
     spark: Optional[SparkSession] = None,
@@ -31,7 +31,7 @@ def read_stream(
 
 @overload
 def read_stream(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     *,
     options: Optional[dict[str, str]] = None,
@@ -40,9 +40,9 @@ def read_stream(
 
 
 def read_stream(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
-    schema_path: Optional[Union[Path, str]] = None,
+    schema_path: Optional[Union[FileSharePath, str]] = None,
     hints: Optional[Union[str, List[str]]] = None,
     schema: Optional[StructType] = None,
     options: Optional[dict[str, str]] = None,
@@ -60,9 +60,9 @@ def read_stream(
 
 
 def _read_stream(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
-    schema_path: Optional[Union[Path, str]] = None,
+    schema_path: Optional[Union[FileSharePath, str]] = None,
     hints: Optional[Union[str, List[str]]] = None,
     schema: Optional[StructType] = None,
     options: Optional[dict[str, str]] = None,
@@ -78,7 +78,7 @@ def _read_stream(
     else:
         file_format = "binaryFile" if file_format == "pdf" else file_format
         if isinstance(src, str):
-            src = Path(src)
+            src = FileSharePath(src)
         if file_format == "delta":
             reader = spark.readStream.format("delta")
         else:
@@ -89,7 +89,8 @@ def _read_stream(
             else:
                 assert schema_path
                 if isinstance(schema_path, str):
-                    schema_path = Path(schema_path)
+                    schema_path = FileSharePath(schema_path)
+
                 reader.option("cloudFiles.inferColumnTypes", "true")
                 reader.option("cloudFiles.useIncrementalListing", "true")
                 reader.option("cloudFiles.schemaEvolutionMode", "addNewColumns")
@@ -117,7 +118,7 @@ def _read_stream(
 
 @overload
 def read_batch(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     schema: StructType,
     options: Optional[dict[str, str]] = None,
@@ -127,7 +128,7 @@ def read_batch(
 
 @overload
 def read_batch(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     *,
     options: Optional[dict[str, str]] = None,
@@ -136,7 +137,7 @@ def read_batch(
 
 
 def read_batch(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     schema: Optional[StructType] = None,
     options: Optional[dict[str, str]] = None,
@@ -152,7 +153,7 @@ def read_batch(
 
 
 def _read_batch(
-    src: Union[Path, str],
+    src: Union[FileSharePath, str],
     file_format: str,
     schema: Optional[StructType] = None,
     options: Optional[dict[str, str]] = None,
@@ -169,7 +170,7 @@ def _read_batch(
         path_glob_filter = file_format
         file_format = "binaryFile" if file_format == "pdf" else file_format
         if isinstance(src, str):
-            src = Path(src)
+            src = FileSharePath(src)
         reader = spark.read.format(file_format)
         reader = reader.option("pathGlobFilter", f"*.{path_glob_filter}")
         if schema:
@@ -201,7 +202,7 @@ def read(
 def read(
     stream: bool,
     *,
-    path: Union[Path, str],
+    path: Union[FileSharePath, str],
     file_format: str = "delta",
     metadata: Optional[bool] = False,
     spark: Optional[SparkSession] = None,
@@ -212,7 +213,7 @@ def read(
 def read(
     stream: bool,
     *,
-    path: Union[Path, str],
+    path: Union[FileSharePath, str],
     file_format: str,
     schema: StructType,
     options: Optional[dict[str, str]] = None,
@@ -225,9 +226,9 @@ def read(
 def read(
     stream: bool,
     *,
-    path: Union[Path, str],
+    path: Union[FileSharePath, str],
     file_format: str,
-    schema_path: Union[Path, str],
+    schema_path: Union[FileSharePath, str],
     options: Optional[dict[str, str]] = None,
     metadata: Optional[bool] = True,
     spark: Optional[SparkSession] = None,
@@ -237,9 +238,9 @@ def read(
 def read(
     stream: bool,
     table: Optional[str] = None,
-    path: Optional[Union[Path, str]] = None,
+    path: Optional[Union[FileSharePath, str]] = None,
     file_format: Optional[str] = None,
-    schema_path: Optional[Union[Path, str]] = None,
+    schema_path: Optional[Union[FileSharePath, str]] = None,
     schema: Optional[StructType] = None,
     hints: Optional[Union[str, List[str]]] = None,
     options: Optional[dict[str, str]] = None,