cloe-nessy 0.2.9__py3-none-any.whl → 0.2.10__py3-none-any.whl

cloe_nessy/file_utilities/get_file_paths.py

@@ -5,7 +5,12 @@ from .factory import FileRetrievalFactory
 from .location_types import LocationType
 
 
-def get_file_paths(location: str, file_name_pattern: str | None = None, search_subdirs: bool = True) -> list[str]:
+def get_file_paths(
+    location: str,
+    file_name_pattern: str | None = None,
+    search_subdirs: bool = True,
+    **kwargs,
+) -> list[str]:
     """Retrieves file paths from a specified location based on the provided criteria.
 
     This function determines the type of location (e.g., local directory, blob storage),
@@ -16,6 +21,7 @@ def get_file_paths(location: str, file_name_pattern: str | None = None, search_s
         location: The location to search for files. This could be a path to a local directory or a URI for blob storage.
         file_name_pattern: The file file_name_pattern to filter by as string. None retrieves all files regardless of file_name_pattern.
         search_subdirs: Whether to include files from subdirectories in the search.
+        kwargs: Additional keyword arguments.
 
     Returns:
         A list of file paths that match the specified criteria. The paths are returned as strings.
@@ -37,7 +43,7 @@ def get_file_paths(location: str, file_name_pattern: str | None = None, search_s
     logger.info(
         f"Retrieving file paths from location [  '{location}'  ] with strategy [  '{strategy.__class__.__name__}'  ]"
     )
-    paths = strategy.get_file_paths(location, file_name_pattern, search_subdirs)
+    paths = strategy.get_file_paths(location, file_name_pattern, search_subdirs, **kwargs)
     logger.debug("paths:", paths)
     return paths
 

cloe_nessy/file_utilities/strategies/base_strategy.py

@@ -11,7 +11,12 @@ class FileRetrievalStrategy(ABC):
 
     @staticmethod
     @abstractmethod
-    def get_file_paths(location: str, extension: str | None = None, search_subdirs: bool = True) -> list[str]:
+    def get_file_paths(
+        location: str,
+        extension: str | None = None,
+        search_subdirs: bool = True,
+        **kwargs,
+    ) -> list[str]:
         """Retrieves a list of file paths based on the specified criteria.
 
         Args:
@@ -19,11 +24,12 @@ class FileRetrievalStrategy(ABC):
             extension: The file extension to filter by. If None, no extension filtering is applied.
                 If an empty string, it matches files with no extension.
             search_subdirs: Whether to search in subdirectories.
+            kwargs: Additional keyword arguments that may be used by concrete implementations
 
         Returns:
             list[str]: A list of file paths that match the specified criteria.
         """
-        pass
+        raise NotImplementedError("Concrete implementations must provide the logic for retrieving file paths.")
 
     @staticmethod
     def _matches_extension(file_name: str, extension: str | None) -> bool:

cloe_nessy/file_utilities/strategies/local_strategy.py

@@ -12,7 +12,12 @@ class LocalDirectoryStrategy(FileRetrievalStrategy):
     """
 
     @staticmethod
-    def get_file_paths(location: str, extension: str | None = None, search_subdirs: bool = True) -> list[str]:
+    def get_file_paths(
+        location: str,
+        extension: str | None = None,
+        search_subdirs: bool = True,
+        **kwargs,  # noqa: ARG004
+    ) -> list[str]:
         """Recursively retrieves all files with a specified extension from a given directory and its subdirectories.
 
         Args:
@@ -20,6 +25,7 @@ class LocalDirectoryStrategy(FileRetrievalStrategy):
             extension: File extension, e.g., 'csv', 'json'. Input an empty string to get files without any
                                     extension, input None to get all files.
             search_subdirs: If True, function will also search within all subdirectories.
+            kwargs: Additional keyword arguments. Used in the OneLakeStrategy.
 
         Returns:
             List: List of files in the directory and its subdirectories with the given extension.

cloe_nessy/file_utilities/strategies/onelake_strategy.py

@@ -6,7 +6,12 @@ class OneLakeStrategy(FileRetrievalStrategy):
     """Strategy for retrieving files from the OneLake."""
 
     @staticmethod
-    def get_file_paths(location: str, extension: str | None = None, search_subdirs: bool = True) -> list:
+    def get_file_paths(
+        location: str,
+        extension: str | None = None,
+        search_subdirs: bool = True,
+        **kwargs,
+    ) -> list:
         """Recursively retrieves all files with a specified extension from a given directory and its subdirectories.
 
         Args:
@@ -14,6 +19,7 @@ class OneLakeStrategy(FileRetrievalStrategy):
             extension: File extension, e.g., 'csv', 'json'. Input an empty string to get files without any
                                     extension, input None to get all files.
             search_subdirs: If True, function will also search within all subdirectories.
+            kwargs: Additional keyword arguments.
 
         Returns:
             List: List of files in the directory and its subdirectories with the given extension.
@@ -27,5 +33,18 @@ class OneLakeStrategy(FileRetrievalStrategy):
 
         file_paths = LocalDirectoryStrategy.get_file_paths(location, extension, search_subdirs)
 
-        shortened_file_paths = [p.replace("/lakehouse/default/", "") for p in file_paths]
-        return shortened_file_paths
+        if kwargs.get("onelake_relative_paths", False) is True:
+            file_paths = OneLakeStrategy._relative_file_paths(file_paths)
+
+        return file_paths
+
+    @staticmethod
+    def _relative_file_paths(file_paths: list[str]) -> list[str]:
+        """OneLake expects relative paths when working with spark.
+
+        Note:
+            Long Paths (in the format '/lakehouse/default/Files/my_file') are
+            used, e.g., when working with Pandas or os.
+        """
+        relative_file_paths = [p.replace("/lakehouse/default/", "") for p in file_paths]
+        return relative_file_paths

cloe_nessy/file_utilities/strategies/utils_strategy.py

@@ -11,7 +11,12 @@ class UtilsStrategy(FileRetrievalStrategy):
     """
 
     @staticmethod
-    def get_file_paths(location: str, extension: str | None = None, search_subdirs: bool = True) -> list:
+    def get_file_paths(
+        location: str,
+        extension: str | None = None,
+        search_subdirs: bool = True,
+        **kwargs,  # noqa: ARG004
+    ) -> list:
         """Recursively retrieves all files with a specified extension from a given directory and its subdirectories.
 
         Args:
@@ -19,6 +24,7 @@ class UtilsStrategy(FileRetrievalStrategy):
             extension: File extension, e.g., 'csv', 'json'. Input an empty string to get files without any
                                     extension, input None to get all files.
             search_subdirs: If True, function will also search within all subdirectories.
+            kwargs: Additional keyword arguments. Used in the OneLakeStrategy.
 
         Returns:
             List: List of files in the directory and its subdirectories with the given extension.

cloe_nessy/integration/reader/file_reader.py

@@ -54,7 +54,7 @@ class FileReader(BaseReader):
             spark_format = extension_to_datatype_dict[extension]
         self._console_logger.debug(f"Reading files with format: {spark_format}")
         if extension:
-            file_paths = get_file_paths(location, extension, search_subdirs)
+            file_paths = get_file_paths(location, extension, search_subdirs, onelake_relative_paths=True)
         else:
             file_paths = [location]
         self._console_logger.debug(f"Found {len(file_paths)} files to read")

cloe_nessy/session/session_manager.py

@@ -62,7 +62,9 @@ class SessionManager:
         return cls._spark
 
     @classmethod
-    def get_utils(cls) -> Any:  # return type should be Union[DBUtils, MsSparkUtils], but can't import locally.
+    def get_utils(
+        cls,
+    ) -> Any:  # return type should be Union[DBUtils, MsSparkUtils], but can't import locally.
         """Get or create a DBUtils or MsSparkUtils instance, depending on the context.
 
         In Databricks this will return DBUtils, while in Fabric it will return MsSparkUtils.
@@ -167,18 +169,22 @@ class SessionManager:
     @classmethod
     def get_spark_builder(cls):
         """Get the SparkSession builder based on the current environment."""
+        cls._detect_env()
         builders = {
             cls.Environment.DATABRICKS_UI: SparkSession.builder,
             cls.Environment.FABRIC_UI: SparkSession.builder,
-            cls.Environment.DATABRICKS_CONNECT: cls._get_databricks_connect_builder(),
-            cls.Environment.OTHER_REMOTE_SPARK: cls._get_databricks_connect_builder(),
+            cls.Environment.DATABRICKS_CONNECT: cls._get_databricks_connect_builder,
+            cls.Environment.OTHER_REMOTE_SPARK: cls._get_databricks_connect_builder,
         }
-
         builder = builders.get(cls._env)
         if builder is None:
             raise ValueError(f"Unsupported environment: {cls._env}")
 
-        return builder
+        match cls._env:
+            case cls.Environment.DATABRICKS_CONNECT | cls.Environment.OTHER_REMOTE_SPARK:
+                return builder()
+            case _:
+                return builder
 
     @staticmethod
     def _get_databricks_connect_builder():

{cloe_nessy-0.2.9.dist-info → cloe_nessy-0.2.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cloe-nessy
-Version: 0.2.9
+Version: 0.2.10
 Summary: Your friendly datalake monster.
 Home-page: https://initions.com/
 Author: initions

{cloe_nessy-0.2.9.dist-info → cloe_nessy-0.2.10.dist-info}/RECORD

@@ -9,20 +9,20 @@ cloe_nessy/clients/api_client/exceptions.py,sha256=VR9nYMHWzIRLlMZMrPpOsEX0X_P0j
 cloe_nessy/file_utilities/__init__.py,sha256=nY8H48jYHvTy0VYSRHVhZaFMlzfch4-T7y3N73tgMpI,73
 cloe_nessy/file_utilities/exceptions.py,sha256=RDeV2S6AQnFhFINRo84HDV_hk2RMrf5oNQ7GhHmAZy0,97
 cloe_nessy/file_utilities/factory.py,sha256=JONYGI8MCkNwG2_ujvjN3iB7BIdl7SqXKgV05YY_i4E,1735
-cloe_nessy/file_utilities/get_file_paths.py,sha256=RoIOaBcHCMPiVHVEDm5ijnhLxFABZNVZK-nXooh2c7A,2895
+cloe_nessy/file_utilities/get_file_paths.py,sha256=wQCNBi7kgM32BSFlCuKFnORd9myjZUygpNm2-tF1F54,2980
 cloe_nessy/file_utilities/location_types.py,sha256=G0FjpEu4_inmWbu5tvs2FyZv2TIhmPgjWU_Rtvmd6i8,801
 cloe_nessy/file_utilities/strategies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloe_nessy/file_utilities/strategies/base_strategy.py,sha256=qOcJE7re9-LNlrwtpFoAQIzH_YXx0jHHNKeqKEZzKrs,2591
-cloe_nessy/file_utilities/strategies/local_strategy.py,sha256=z_fKAfPcAeKE7SJ8-v8iFwHiWBxyFipfPs7VJhv1FSU,2073
-cloe_nessy/file_utilities/strategies/onelake_strategy.py,sha256=dIAkHPbmybVVxZSlnCPx3CpprahtopV5lxjJXPUKhz8,1405
-cloe_nessy/file_utilities/strategies/utils_strategy.py,sha256=luFPpjdc-B9-vcLoHun9gguG86o0ERzH9lplmzOpoQE,3050
+cloe_nessy/file_utilities/strategies/base_strategy.py,sha256=2BdGdP8ThjIP4e_fv7apx7Hg_L6q3nsPdek4oPgN7CI,2833
+cloe_nessy/file_utilities/strategies/local_strategy.py,sha256=6OcEjzLvRTBT8FKXhkLI0befT48SHutGHFIXMq5Sq8E,2217
+cloe_nessy/file_utilities/strategies/onelake_strategy.py,sha256=RnQjWtWIFzFj-zPqzyZaPYIjtjXkgP-K7-VA8GhkNmg,1980
+cloe_nessy/file_utilities/strategies/utils_strategy.py,sha256=urayKfOUpSaXKgTs1KVK0TS7FWVrJ3k4OLKh35sCxAU,3194
 cloe_nessy/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/integration/reader/__init__.py,sha256=J5vlORqHLBpHEvzIwfIjzN5xEdOat-8jlmdLcGj8nsA,239
 cloe_nessy/integration/reader/api_reader.py,sha256=j3Z5O1oH-Zc43TyA_aYtnDNYC9xFMxMqDsRQWtEZGD8,5636
 cloe_nessy/integration/reader/catalog_reader.py,sha256=tGK-Y0jZQGOrF9eZUzSr7ils-L58uex6qH9PZ81ZLy8,1835
 cloe_nessy/integration/reader/excel_reader.py,sha256=4kifpIakHpGmap0-P0SUgjJoQdY-eeiZBIDrQp87wK8,8012
 cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
-cloe_nessy/integration/reader/file_reader.py,sha256=CsKjn2W7-w6drbWtD7PNMniJ8kCgxm4qW4knhYA37tg,3850
+cloe_nessy/integration/reader/file_reader.py,sha256=pkrW_N5avqQpqcZuIQgHw5CFf7DFpSuKvq88zPZPfyY,3879
 cloe_nessy/integration/reader/reader.py,sha256=e2KVPePQme8SBQJEbL-3zpGasOgTiEvKFTslow2wGPw,1034
 cloe_nessy/integration/writer/__init__.py,sha256=NIh0t1RYlG3J1Y5_CvnR36N9tISmcElD5Tq06ksmqoA,71
 cloe_nessy/integration/writer/catalog_writer.py,sha256=49lDvYttUY79Ye_OMN2cji7lGJNNML4TTsjY7VvLVfc,2137
@@ -67,12 +67,12 @@ cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=Kez8puDK7cRfhleBE
 cloe_nessy/pipeline/actions/transform_union.py,sha256=TDER06IABzxvIez4bGLKCLaDA4eScpTzYRbfUzwv_RQ,2342
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=6yAHTX5kZviumgBW_NYVGAUin6U2nDzmic9of6wA8FY,2590
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=B1TCfpZ8aieN37WWyY2b9qs2U7muyL1edzDCCPeOxHs,6407
+cloe_nessy/session/session_manager.py,sha256=7LNerwILGkgt752cZLs2nlABGWiaoKdmOuLGWHZ6uYQ,6618
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.2.9.dist-info/METADATA,sha256=42HBrdWGyYtb3eOER40KHnW0wVOUK0yQHI8Xi5uEscE,1837
-cloe_nessy-0.2.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-cloe_nessy-0.2.9.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.2.9.dist-info/RECORD,,
+cloe_nessy-0.2.10.dist-info/METADATA,sha256=W9E01GNme6Zst17uy9TAW_eP7FL_Ng-HkKaUvXf8838,1838
+cloe_nessy-0.2.10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+cloe_nessy-0.2.10.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.2.10.dist-info/RECORD,,