daplapath 2.0.9__py3-none-any.whl → 2.0.11__py3-none-any.whl

daplapath/path.py

@@ -31,6 +31,11 @@ try:
 except ImportError:
     pass
 
+try:
+    from google.cloud import storage
+except ImportError:
+    pass
+
 # regex with the prefix '_v' followed by an integer of any length
 VERSION_PATTERN = r"_v(\d+)\."
 VERSION_PREFIX = "_v"
@@ -113,15 +118,26 @@ class LocalFileSystem(AbstractFileSystem):
         return os.makedirs(path, exist_ok=exist_ok)
 
 
-class GCSFileSystem(gcsfs.GCSFileSystem):
+class MyGCSFileSystem(gcsfs.GCSFileSystem):
     def isdir(self, path: str) -> bool:
         """Check if path is a directory."""
         info = super(gcsfs.GCSFileSystem, self).info(path)
         return info["type"] == "directory"
 
+    def rmdir(self, path: str) -> None:
+        """Remove contents of a directory in GCS. It might take some time before files are actually deleted."""
+        path = pathlib.Path(path)
+        remaining = self.glob(str(path / "**"))
+        assert all(self.isdir(x) for x in remaining), remaining
+        storage_client = storage.Client()
+        bucket = storage_client.get_bucket(path.parts[0])
+        blobs = bucket.list_blobs(prefix="/".join(path.parts) + "/")
+        for blob in blobs:
+            blob.delete()
+
 
 if any("dapla" in key.lower() for key in os.environ) and "gcsfs" in locals():
-    _config = Config(GCSFileSystem)
+    _config = Config(MyGCSFileSystem)
 else:
     _config = Config(LocalFileSystem)
 
@@ -314,7 +330,14 @@ class Path(str, _PathBase):
         return self._cp_or_mv(destination, "cp")
 
     def mv(self, destination: "Path | str") -> "Path":
-        return self._cp_or_mv(destination, "mv")
+        was_dir = self.isdir()
+        out_path = self._cp_or_mv(destination, "mv")
+        if was_dir:
+            try:
+                self.file_system.rmdir(str(self))
+            except (FileNotFoundError, NotADirectoryError):
+                pass
+        return out_path
 
     def versions(self, include_versionless: bool = False) -> "PathSeries":
         """Returns a PathSeries of all versions of the file."""
@@ -393,7 +416,9 @@ class Path(str, _PathBase):
         if timeout:
             timestamp: datetime.datetime = highest_numbered.timestamp
 
-            time_should_be_at_least = pd.Timestamp.now() - pd.Timedelta(minutes=timeout)
+            time_should_be_at_least = pd.Timestamp.now(tz="Europe/Oslo").replace(
+                tzinfo=None
+            ).round("s") - pd.Timedelta(minutes=timeout)
             if timestamp > time_should_be_at_least:
                 raise ValueError(
                     f"Latest version of the file was updated {timestamp[0]}, which "
@@ -483,6 +508,8 @@ class Path(str, _PathBase):
         """
         if not isinstance(period, (str, int)):
             raise TypeError(f"'period' should be string or int. Got {type(period)}")
+        if not self.period:
+            raise ValueError(f"Cannot set period to path without period. {self}")
         if str(period) == self.period:
             return self
         return self.with_periods(period)
@@ -506,6 +533,8 @@ class Path(str, _PathBase):
             raise TypeError(
                 f"'to_period' should be string or int. Got {type(to_period)}"
             )
+        if not self.periods:
+            raise ValueError(f"Cannot set period to path without period. {self}")
 
         periods: tuple[str] = (
             (str(from_period), str(to_period)) if to_period else (str(from_period),)
@@ -696,9 +725,19 @@ class Path(str, _PathBase):
 
     @property
     def partition_root(self) -> "Path":
-        if ".parquet" not in self:
+        if not self.suffix or self.count(self.suffix) != 2:
             return self
-        return self.split(".parquet")[0] + ".parquet"
+        return self._new(self.split(self.suffix)[0] + self.suffix)
+
+    def is_partitioned(self) -> bool:
+        if (
+            not self.suffix
+            or self.suffix not in self
+            or self.isfile()
+            and self.count(self.suffix) != 2
+        ):
+            return False
+        return bool(len(self.glob(f"**/*{self.suffix}")))
 
     def isfile(self) -> bool:
         return not self.isdir()
@@ -971,32 +1010,44 @@ class PathSeries(pd.Series, _PathBase):
 
     def within_minutes(self, minutes: int):
         """Select files with a timestamp within the given number of minutes."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(minutes=minutes)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(minutes=minutes)
         return self.files[lambda x: x.timestamp > time_then]
 
     def within_hours(self, hours: int):
         """Select files with a timestamp within the given number of hours."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(hours=hours)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(hours=hours)
         return self.files[lambda x: x.timestamp > time_then]
 
     def within_days(self, days: int):
         """Select files with a timestamp within the given number of days."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(days=days)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(days=days)
         return self.files[lambda x: x.timestamp > time_then]
 
     def not_within_minutes(self, minutes: int):
         """Select files with a timestamp within the given number of minutes."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(minutes=minutes)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(minutes=minutes)
         return self.files[lambda x: x.timestamp < time_then]
 
     def not_within_hours(self, hours: int):
         """Select files with a timestamp within the given number of hours."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(hours=hours)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(hours=hours)
         return self.files[lambda x: x.timestamp < time_then]
 
     def not_within_days(self, days: int):
         """Select files with a timestamp within the given number of days."""
-        time_then = pd.Timestamp.now() - pd.Timedelta(days=days)
+        time_then = pd.Timestamp.now(tz="Europe/Oslo").replace(tzinfo=None).round(
+            "s"
+        ) - pd.Timedelta(days=days)
         return self.files[lambda x: x.timestamp < time_then]
 
     @property

{daplapath-2.0.9.dist-info → daplapath-2.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: daplapath
-Version: 2.0.9
+Version: 2.0.11
 Summary: A pathlib.Path class for dapla
 License: MIT
 Author: ort

daplapath-2.0.11.dist-info/RECORD

@@ -0,0 +1,6 @@
+daplapath/__init__.py,sha256=Qdpwhl8y3-i_42-4-KVT-sPQtJqwWRENLFxROZ_rfbU,86
+daplapath/path.py,sha256=kGfYhBgA0eZLq26wWpM7XUbh1-xsN-KdamXsevXwbu8,56058
+daplapath-2.0.11.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
+daplapath-2.0.11.dist-info/METADATA,sha256=NRXrbquGmzccuj6XsFbvPZfyJZrt-ANfG-BsjO8Mb7U,14698
+daplapath-2.0.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+daplapath-2.0.11.dist-info/RECORD,,

daplapath-2.0.9.dist-info/RECORD

@@ -1,6 +0,0 @@
-daplapath/__init__.py,sha256=Qdpwhl8y3-i_42-4-KVT-sPQtJqwWRENLFxROZ_rfbU,86
-daplapath/path.py,sha256=8XFAPMmiDKxX_XLOORFJPmNf5BIkyf6drMdG--CwI94,54190
-daplapath-2.0.9.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
-daplapath-2.0.9.dist-info/METADATA,sha256=jM7z7Rgzfh2gAe6rgqA_WY0EVjr3KsDOYCDo0JcyvJY,14697
-daplapath-2.0.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-daplapath-2.0.9.dist-info/RECORD,,