daplapath 2.0.7__tar.gz → 2.0.9__tar.gz

{daplapath-2.0.7 → daplapath-2.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: daplapath
-Version: 2.0.7
+Version: 2.0.9
 Summary: A pathlib.Path class for dapla
 License: MIT
 Author: ort
@@ -35,14 +35,14 @@ from daplapath.path import Path
 
 
 ```python
-folder = Path('ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024')
+folder = Path('ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024')
 folder
 ```
 
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -107,7 +107,7 @@ file
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -119,7 +119,7 @@ file.parent
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -135,8 +135,7 @@ file.columns
 
 
 
-    Index(['OBJTYPE', 'NAVN', "komm_nr", "fylke_nr", 'AREAL_GDB', 'SHAPE_Length',
-           'SHAPE_Area', 'geometry'],
+    Index(['objtype', 'navn', "komm_nr", "fylke_nr", 'areal_gdb', 'geometry'],
           dtype='object')
 
 
@@ -149,13 +148,11 @@ file.dtypes
 
 
 
-    OBJTYPE         string
-    NAVN            string
+    objtype         string
+    navn            string
     komm_nr       string
     fylke_nr           string
-    AREAL_GDB       double
-    SHAPE_Length    double
-    SHAPE_Area      double
+    areal_gdb       double
     geometry        binary
     dtype: object
 
@@ -206,7 +203,7 @@ file.latest_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -218,7 +215,7 @@ file.highest_numbered_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -231,7 +228,7 @@ file.new_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
 
 
 
@@ -268,11 +265,11 @@ Filtre med hyperlenke. Gjør at man kopierer stien når man klikker på den.
 
 ```python
 print(
-    Path("ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data").tree()
+    Path("ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data").tree()
 )
 ```
 
-    ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data /
+    ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data /
         └──2000 /
             └──SSB_tettsted_flate_p2000.parquet
             └──SSB_tettsted_flate_p2000_v1.parquet

{daplapath-2.0.7 → daplapath-2.0.9}/README.md

@@ -19,14 +19,14 @@ from daplapath.path import Path
 
 
 ```python
-folder = Path('ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024')
+folder = Path('ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024')
 folder
 ```
 
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -91,7 +91,7 @@ file
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -103,7 +103,7 @@ file.parent
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024'
 
 
 
@@ -119,8 +119,7 @@ file.columns
 
 
 
-    Index(['OBJTYPE', 'NAVN', "komm_nr", "fylke_nr", 'AREAL_GDB', 'SHAPE_Length',
-           'SHAPE_Area', 'geometry'],
+    Index(['objtype', 'navn', "komm_nr", "fylke_nr", 'areal_gdb', 'geometry'],
           dtype='object')
 
 
@@ -133,13 +132,11 @@ file.dtypes
 
 
 
-    OBJTYPE         string
-    NAVN            string
+    objtype         string
+    navn            string
     komm_nr       string
     fylke_nr           string
-    AREAL_GDB       double
-    SHAPE_Length    double
-    SHAPE_Area      double
+    areal_gdb       double
     geometry        binary
     dtype: object
 
@@ -190,7 +187,7 @@ file.latest_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -202,7 +199,7 @@ file.highest_numbered_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v1.parquet'
 
 
 
@@ -215,7 +212,7 @@ file.new_version()
 
 
 
-    'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
+    'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2024/ABAS_kommune_utenhav_p2024_v2.parquet'
 
 
 
@@ -252,11 +249,11 @@ Filtre med hyperlenke. Gjør at man kopierer stien når man klikker på den.
 
 ```python
 print(
-    Path("ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data").tree()
+    Path("ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data").tree()
 )
 ```
 
-    ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data /
+    ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data /
         └──2000 /
             └──SSB_tettsted_flate_p2000.parquet
             └──SSB_tettsted_flate_p2000_v1.parquet

{daplapath-2.0.7 → daplapath-2.0.9}/daplapath/path.py

@@ -96,7 +96,7 @@ class LocalFileSystem(AbstractFileSystem):
         return cls.cp_file(source, destination, **kwargs)
 
     @staticmethod
-    def cp_file(self, path1, path2, **kwargs):
+    def cp_file(source, destination, **kwargs):
         os.makedirs(pathlib.Path(destination).parent, exist_ok=True)
         return shutil.copy2(source, destination, **kwargs)
 
@@ -252,11 +252,16 @@ class Path(str, _PathBase):
     def rglob(self, pattern: str, **kwargs) -> "PathSeries":
         return self.glob(pattern, recursive=True, **kwargs)
 
-    def glob(self, pattern: str, recursive: bool = True, **kwargs) -> "PathSeries":
+    def glob(
+        self, pattern: str | None = None, recursive: bool = True, **kwargs
+    ) -> "PathSeries":
         """Create PathSeries of files/directories that match the pattern."""
         recursive = kwargs.get("recurse_symlinks", recursive)
 
-        pattern = str(self / pattern)
+        if pattern:
+            pattern = str(self / pattern)
+        else:
+            pattern = str(self)
 
         # pop kwargs going into PathSeries initialiser.
         iterable_init_args = get_arguments(self._iterable_type)
@@ -300,6 +305,17 @@ class Path(str, _PathBase):
         """
         return self.glob("**", recursive=recursive, **kwargs)
 
+    def rmdir(self) -> None:
+        files = self.glob("**").files
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(self.file_system.rm_file, files))
+
+    def cp(self, destination: "Path | str") -> "Path":
+        return self._cp_or_mv(destination, "cp")
+
+    def mv(self, destination: "Path | str") -> "Path":
+        return self._cp_or_mv(destination, "mv")
+
     def versions(self, include_versionless: bool = False) -> "PathSeries":
         """Returns a PathSeries of all versions of the file."""
         files_in_folder: Iterable[Path] = self.parent.glob("**", recursive=False)
@@ -583,7 +599,7 @@ class Path(str, _PathBase):
 
     @property
     def index_column_names(self) -> list[str]:
-        return _get_index_cols(self.schema)
+        return _get_index_cols(self.schema, self)
 
     @property
     def columns(self) -> pd.Index:
@@ -596,7 +612,7 @@ class Path(str, _PathBase):
             ]
         except (KeyError, TypeError):
             names = schema.names
-        index_cols = _get_index_cols(schema)
+        index_cols = _get_index_cols(schema, self)
         return pd.Index(names).difference(index_cols)
 
     @property
@@ -605,16 +621,14 @@ class Path(str, _PathBase):
         try:
             with self.open("rb") as file:
                 return get_schema(file)
-        except (
-            Exception
-        ):  # (PermissionError, FileNotFoundError, TypeError, IsADirectoryError):
+        except Exception:
             return get_schema(self)
 
     @property
     def dtypes(self) -> pd.Series:
         """Date types of the file's columns."""
         schema = self.schema
-        index_cols = _get_index_cols(schema)
+        index_cols = _get_index_cols(schema, self)
         return pd.Series(schema.types, index=schema.names).loc[
             lambda x: ~x.index.isin(index_cols)
         ]
@@ -625,9 +639,7 @@ class Path(str, _PathBase):
         try:
             with self.open("rb") as file:
                 return get_shape(file)
-        except (
-            Exception
-        ):  # (PermissionError, FileNotFoundError, TypeError, IsADirectoryError):
+        except Exception:
             return get_shape(self)
 
     @property
@@ -722,10 +734,10 @@ class Path(str, _PathBase):
 
         Example
         -------
-        >>> folder = 'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2023'
+        >>> folder = 'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2023'
         >>> file_path = folder / "ABAS_kommune_flate_p2023_v1.parquet"
         >>> file_path
-        'ssb-kart-data-delt-geo-prod/analyse_data/klargjorte-data/2023/ABAS_kommune_flate_p2023_v1.parquet'
+        'ssb-areal-data-delt-kart-prod/analyse_data/klargjorte-data/2023/ABAS_kommune_flate_p2023_v1.parquet'
         """
         if not isinstance(other, (str, PurePath, os.PathLike)):
             raise TypeError(
@@ -783,6 +795,27 @@ class Path(str, _PathBase):
     def _new(self, new_path: str | Path) -> "Path":
         return self.__class__(new_path, self.file_system)
 
+    def _cp_or_mv(self, destination: "Path | str", attr: str) -> "Path":
+        func: Callable = getattr(self.file_system, attr)
+        try:
+            func(self, destination)
+        except FileNotFoundError:
+            destination = self.__class__(destination)
+            sources = list(self.glob("**").files)
+            destinations = [path.replace(self, destination) for path in sources]
+            with ThreadPoolExecutor() as executor:
+                list(executor.map(func, sources, destinations))
+        self._new(destination)
+
+    def keep_newest_partitions(self) -> "Path":
+        def _keep_newest(path):
+            while True:
+                if path.isfile():
+                    pass
+
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(_keep_newest, self.ls()))
+
 
 class PathSeries(pd.Series, _PathBase):
     """A pandas Series for working with GCS (Google Cloud Storage) paths.
@@ -1480,8 +1513,11 @@ def get_path_tree(
     return tree
 
 
-def _get_index_cols(schema: pyarrow.Schema) -> list[str]:
-    cols = json.loads(schema.metadata[b"pandas"])["index_columns"]
+def _get_index_cols(schema: pyarrow.Schema, path_or_file: str | Path) -> list[str]:
+    try:
+        cols = json.loads(schema.metadata[b"pandas"])["index_columns"]
+    except KeyError as e:
+        raise KeyError(f"{e}. For {type(path_or_file)}: {path_or_file}")
     return [x for x in cols if not isinstance(x, dict)]
 
 
@@ -1526,29 +1562,34 @@ def get_schema(file) -> pyarrow.Schema:
         def _get_schema(path):
             try:
                 return pq.read_schema(path)
-            except FileNotFoundError:
-                with file_system.open(path, "rb") as f:
-                    return pq.read_schema(f)
+            except FileNotFoundError as e:
+                try:
+                    with file_system.open(path, "rb") as f:
+                        return pq.read_schema(f)
+                except Exception as e2:
+                    raise e2.__class__(f"{e2}. {path}") from e
+
+        child_paths = file_system.glob(file + "/**/*.parquet")
+        if not len(child_paths):
+            raise e.__class__(f"{e}: {file}") from e
 
         with ThreadPoolExecutor() as executor:
-            return pyarrow.unify_schemas(
-                list(
-                    executor.map(_get_schema, file_system.glob(file + "/**/*.parquet"))
-                ),
-                promote_options="permissive",
+            schemas: list[pyarrow.Schema] = list(
+                executor.map(_get_schema, file_system.glob(file + "/**/*.parquet"))
             )
+        if not schemas:
+            raise ValueError(f"Couldn't find any schemas among {child_paths}.") from e
+
+        return pyarrow.unify_schemas(
+            schemas,
+            promote_options="permissive",
+        )
 
 
 def get_num_rows(file):
     try:
         return pq.read_metadata(file).num_rows
-    except (
-        PermissionError,
-        pyarrow.ArrowInvalid,
-        FileNotFoundError,
-        TypeError,
-        OSError,
-    ) as e:
+    except Exception as e:
         try:
             return ds.dataset(file).count_rows()
         except Exception as e2:
@@ -1565,7 +1606,7 @@ def get_num_rows(file):
 
 def get_shape(file) -> tuple[int, int]:
     schema = get_schema(file)
-    index_cols = _get_index_cols(schema)
+    index_cols = _get_index_cols(schema, file)
     ncol: int = sum(name not in index_cols for name in schema.names)
     nrow: int = get_num_rows(file)
     return nrow, ncol

{daplapath-2.0.7 → daplapath-2.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daplapath"
-version = "2.0.7"
+version = "2.0.9"
 description = "A pathlib.Path class for dapla"
 authors = ["ort <ort@ssb.no>"]
 license = "MIT"