daplapath 2.0.11__tar.gz → 2.0.13__tar.gz

{daplapath-2.0.11 → daplapath-2.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: daplapath
-Version: 2.0.11
+Version: 2.0.13
 Summary: A pathlib.Path class for dapla
 License: MIT
 Author: ort

{daplapath-2.0.11 → daplapath-2.0.13}/daplapath/path.py

@@ -128,7 +128,9 @@ class MyGCSFileSystem(gcsfs.GCSFileSystem):
         """Remove contents of a directory in GCS. It might take some time before files are actually deleted."""
         path = pathlib.Path(path)
         remaining = self.glob(str(path / "**"))
-        assert all(self.isdir(x) for x in remaining), remaining
+        assert all(self.isdir(x) for x in remaining), [
+            x for x in remaining if not self.isdir(x)
+        ]
         storage_client = storage.Client()
         bucket = storage_client.get_bucket(path.parts[0])
         blobs = bucket.list_blobs(prefix="/".join(path.parts) + "/")
@@ -272,6 +274,7 @@ class Path(str, _PathBase):
         self, pattern: str | None = None, recursive: bool = True, **kwargs
     ) -> "PathSeries":
         """Create PathSeries of files/directories that match the pattern."""
+
         recursive = kwargs.get("recurse_symlinks", recursive)
 
         if pattern:
@@ -730,15 +733,31 @@ class Path(str, _PathBase):
         return self._new(self.split(self.suffix)[0] + self.suffix)
 
     def is_partitioned(self) -> bool:
-        if (
-            not self.suffix
-            or self.suffix not in self
-            or self.isfile()
-            and self.count(self.suffix) != 2
-        ):
+        if not self.suffix or self.count(self.suffix) != 2:
             return False
         return bool(len(self.glob(f"**/*{self.suffix}")))
 
+    def has_all_partitions(
+        self, ids: Iterable[str], nrow_min: int, id_col: str, strict: bool = True
+    ) -> bool:
+        if not self.exists():
+            return False
+        paths = self.glob(f"**/*{self.suffix}")
+        for id_ in ids:
+            these_paths = [path for path in paths if f"{id_col}={id_}" in path]
+            if len(these_paths) != 1:
+                return False
+            this_path = next(iter(these_paths))
+            if Path(this_path).nrow < nrow_min:
+                return False
+        if not strict:
+            return True
+        for path in paths:
+            if not any(f"{id_col}={id_}" for id_ in ids):
+                # partition is not in list of ids
+                return False
+        return True
+
     def isfile(self) -> bool:
         return not self.isdir()
 
@@ -971,9 +990,13 @@ class PathSeries(pd.Series, _PathBase):
         """Select only the files in the Series."""
         return self[self.is_file()]
 
-    @property
-    def partition_root(self) -> "PathSeries":
-        return self.files.apply(lambda x: x.partition_root).drop_duplicates()
+    def buckets_path(self) -> "PathSeries":
+        return self.files.apply(lambda x: x.buckets_path())
+
+    def partition_root(self, keep: str | None = "last") -> "PathSeries":
+        return self.files.apply(lambda x: x.partition_root)[
+            lambda x: ~x.duplicated(keep=keep)
+        ]
 
     @property
     def partitioned_files(self) -> "PathSeries":
@@ -1178,9 +1201,18 @@ class PathSeries(pd.Series, _PathBase):
 
     def __getattr__(self, attr: str) -> Any:
         """Get Path attribute for each row."""
+
+        def get_property(path: Path):
+            x = getattr(path, attr)
+            if callable(x):
+                raise ValueError(
+                    f"{self.__class__.__name__} cannot access Path methods, only properties."
+                )
+            return x
+
         if attr in dir(self._path_type) and attr != "name" and attr not in dir(str):
             try:
-                series = self.apply(lambda path: getattr(path, attr))
+                series = self.apply(get_property)
                 series.name = attr
                 return series
             except IndexError as e:

{daplapath-2.0.11 → daplapath-2.0.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "daplapath"
-version = "2.0.11"
+version = "2.0.13"
 description = "A pathlib.Path class for dapla"
 authors = ["ort <ort@ssb.no>"]
 license = "MIT"