daplapath 2.0.7__py3-none-any.whl → 2.0.8__py3-none-any.whl

daplapath/path.py

@@ -96,7 +96,7 @@ class LocalFileSystem(AbstractFileSystem):
         return cls.cp_file(source, destination, **kwargs)
 
     @staticmethod
-    def cp_file(self, path1, path2, **kwargs):
+    def cp_file(source, destination, **kwargs):
         os.makedirs(pathlib.Path(destination).parent, exist_ok=True)
         return shutil.copy2(source, destination, **kwargs)
 
@@ -252,11 +252,16 @@ class Path(str, _PathBase):
     def rglob(self, pattern: str, **kwargs) -> "PathSeries":
         return self.glob(pattern, recursive=True, **kwargs)
 
-    def glob(self, pattern: str, recursive: bool = True, **kwargs) -> "PathSeries":
+    def glob(
+        self, pattern: str | None = None, recursive: bool = True, **kwargs
+    ) -> "PathSeries":
         """Create PathSeries of files/directories that match the pattern."""
         recursive = kwargs.get("recurse_symlinks", recursive)
 
-        pattern = str(self / pattern)
+        if pattern:
+            pattern = str(self / pattern)
+        else:
+            pattern = str(self)
 
         # pop kwargs going into PathSeries initialiser.
         iterable_init_args = get_arguments(self._iterable_type)
@@ -300,6 +305,17 @@ class Path(str, _PathBase):
         """
         return self.glob("**", recursive=recursive, **kwargs)
 
+    def rmdir(self) -> None:
+        files = self.glob("**").files
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(self.file_system.rm_file, files))
+
+    def cp(self, destination: "Path | str") -> "Path":
+        return self._cp_or_mv(destination, "cp")
+
+    def mv(self, destination: "Path | str") -> "Path":
+        return self._cp_or_mv(destination, "mv")
+
     def versions(self, include_versionless: bool = False) -> "PathSeries":
         """Returns a PathSeries of all versions of the file."""
         files_in_folder: Iterable[Path] = self.parent.glob("**", recursive=False)
@@ -722,10 +738,10 @@ class Path(str, _PathBase):
 
         Example
         -------
-        >>> folder = 'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2023'
+        >>> folder = 'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2023'
         >>> file_path = folder / "ABAS_kommune_flate_p2023_v1.parquet"
         >>> file_path
-        'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2023/ABAS_kommune_flate_p2023_v1.parquet'
+        'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2023/ABAS_kommune_flate_p2023_v1.parquet'
         """
         if not isinstance(other, (str, PurePath, os.PathLike)):
             raise TypeError(
@@ -783,6 +799,27 @@ class Path(str, _PathBase):
     def _new(self, new_path: str | Path) -> "Path":
         return self.__class__(new_path, self.file_system)
 
+    def _cp_or_mv(self, destination: "Path | str", attr: str) -> "Path":
+        func: Callable = getattr(self.file_system, attr)
+        try:
+            func(self, destination)
+        except FileNotFoundError:
+            destination = self.__class__(destination)
+            sources = list(self.glob("**").files)
+            destinations = [path.replace(self, destination) for path in sources]
+            with ThreadPoolExecutor() as executor:
+                list(executor.map(func, sources, destinations))
+        self._new(destination)
+
+    def keep_newest_partitions(self) -> "Path":
+        def _keep_newest(path):
+            while True:
+                if path.isfile():
+                    pass
+
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(_keep_newest, self.ls()))
+
 
 class PathSeries(pd.Series, _PathBase):
     """A pandas Series for working with GCS (Google Cloud Storage) paths.
@@ -1526,9 +1563,12 @@ def get_schema(file) -> pyarrow.Schema:
         def _get_schema(path):
             try:
                 return pq.read_schema(path)
-            except FileNotFoundError:
-                with file_system.open(path, "rb") as f:
-                    return pq.read_schema(f)
+            except FileNotFoundError as e:
+                try:
+                    with file_system.open(path, "rb") as f:
+                        return pq.read_schema(f)
+                except Exception as e2:
+                    raise e2.__class__(f"{e2}. {path}") from e
 
         with ThreadPoolExecutor() as executor:
             return pyarrow.unify_schemas(
@@ -1542,13 +1582,7 @@ def get_schema(file) -> pyarrow.Schema:
 def get_num_rows(file):
     try:
         return pq.read_metadata(file).num_rows
-    except (
-        PermissionError,
-        pyarrow.ArrowInvalid,
-        FileNotFoundError,
-        TypeError,
-        OSError,
-    ) as e:
+    except Exception as e:
         try:
             return ds.dataset(file).count_rows()
         except Exception as e2:

{daplapath-2.0.7.dist-info → daplapath-2.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: daplapath
-Version: 2.0.7
+Version: 2.0.8
 Summary: A pathlib.Path class for dapla
 License: MIT
 Author: ort
@@ -35,14 +35,14 @@ from daplapath.path import Path
 
 
 ```python
-folder = Path('ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024')
+folder = Path('ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024')
 folder
 ```
 
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -107,7 +107,7 @@ file
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -119,7 +119,7 @@ file.parent
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -135,8 +135,7 @@ file.columns
 
 
 
-    Index(['OBJTYPE', 'NAVN', "komm_nr", "fylke_nr", 'AREAL_GDB', 'SHAPE_Length',
-           'SHAPE_Area', 'geometry'],
+    Index(['objtype', 'navn', "komm_nr", "fylke_nr", 'areal_gdb', 'geometry'],
           dtype='object')
 
 
@@ -149,13 +148,11 @@ file.dtypes
 
 
 
-    OBJTYPE         string
-    NAVN            string
+    objtype         string
+    navn            string
     komm_nr       string
     fylke_nr           string
-    AREAL_GDB       double
-    SHAPE_Length    double
-    SHAPE_Area      double
+    areal_gdb       double
     geometry        binary
     dtype: object
 
@@ -206,7 +203,7 @@ file.latest_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -218,7 +215,7 @@ file.highest_numbered_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -231,7 +228,7 @@ file.new_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
 
 
 
@@ -268,11 +265,11 @@ Filtre med hyperlenke. Gjør at man kopierer stien når man klikker på den.
 
 ```python
 print(
-    Path("ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data").tree()
+    Path("ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data").tree()
 )
 ```
 
-    ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data /
+    ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data /
         └──2000 /
             └──SSB_tettsted_flate_p2000.parquet
             └──SSB_tettsted_flate_p2000_v1.parquet

daplapath-2.0.8.dist-info/RECORD

@@ -0,0 +1,6 @@
+daplapath/__init__.py,sha256=Qdpwhl8y3-i_42-4-KVT-sPQtJqwWRENLFxROZ_rfbU,86
+daplapath/path.py,sha256=f4N8JeJ-RqkS0jIoAk69FlVTGR8kyj64FhJR59CZX1w,53928
+daplapath-2.0.8.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
+daplapath-2.0.8.dist-info/METADATA,sha256=qc6AKIcFCJTun5HaeakY80u03SBsNv0Okzi2kJORvAg,14697
+daplapath-2.0.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+daplapath-2.0.8.dist-info/RECORD,,

daplapath-2.0.7.dist-info/RECORD

@@ -1,6 +0,0 @@
-daplapath/__init__.py,sha256=Qdpwhl8y3-i_42-4-KVT-sPQtJqwWRENLFxROZ_rfbU,86
-daplapath/path.py,sha256=_6qbMdkEis1dNsoXwCdl1IT2hedTORvR5vlDNbxPNak,52602
-daplapath-2.0.7.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
-daplapath-2.0.7.dist-info/METADATA,sha256=mYwZhDxhkjptdi8K2s8oHLSMIzGfSVbPX7JB1mcs-cA,14774
-daplapath-2.0.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-daplapath-2.0.7.dist-info/RECORD,,