megfile 4.2.4__py3-none-any.whl → 4.2.5__py3-none-any.whl

megfile/__init__.py

@@ -211,6 +211,11 @@ try:
 except ImportError:
     Sftp2Path = None
 
+try:
+    from megfile.webdav_path import WebdavPath
+except ImportError:
+    WebdavPath = None
+
 __all__ = [
     "smart_access",
     "smart_cache",

megfile/fs.py

@@ -1,7 +1,7 @@
 import os
 from stat import S_ISDIR as stat_isdir
 from stat import S_ISLNK as stat_islnk
-from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
+from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
 
 from megfile.fs_path import (
     FSPath,
@@ -52,6 +52,7 @@ __all__ = [
     "fs_islink",
     "fs_ismount",
     "fs_save_as",
+    "fs_open",
 ]
 
 
@@ -612,3 +613,15 @@ def fs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> N
     :param overwrite: whether or not overwrite file when exists
     """
     return fs_rename(src_path, dst_path, overwrite)
+
+
+def fs_open(path: PathLike, mode: str = "r", **kwargs) -> IO:
+    """
+    Open file on fs
+
+    :param path: Given path
+    :param mode: File open mode, like built-in open function
+    :param buffering: Buffering policy, like built-in open function
+    :returns: A file-like object
+    """
+    return FSPath(path).open(mode, **kwargs)

megfile/fs_path.py

@@ -17,6 +17,7 @@ from megfile.interfaces import (
     Access,
     ContextIterator,
     FileEntry,
+    FileLike,
     PathLike,
     StatResult,
     URIPath,
@@ -85,6 +86,36 @@ def _fs_rename_file(
     shutil.move(src_path, dst_path)
 
 
+class WrapAtomic(FileLike):
+    __atomic__ = True
+
+    def __init__(self, fileobj):
+        self.fileobj = fileobj
+        self.temp_name = f"{self.name}.temp"
+        os.rename(self.name, self.temp_name)
+
+    @property
+    def name(self):
+        return self.fileobj.name
+
+    @property
+    def mode(self):
+        return self.fileobj.mode
+
+    def _close(self):
+        self.fileobj.close()
+        os.rename(self.temp_name, self.name)
+
+    def _abort(self):
+        try:
+            os.unlink(self.temp_name)
+        except FileNotFoundError:
+            pass
+
+    def __getattr__(self, name: str):
+        return getattr(self.fileobj, name)
+
+
 @SmartPath.register
 class FSPath(URIPath):
     """file protocol
@@ -627,9 +658,11 @@ class FSPath(URIPath):
         """
         self._check_int_path()
 
-        if missing_ok and not self.exists():
-            return
-        os.unlink(self.path_without_protocol)  # pyre-ignore[6]
+        try:
+            os.unlink(self.path_without_protocol)  # pyre-ignore[6]
+        except FileNotFoundError:
+            if not missing_ok:
+                raise
 
     def walk(
         self, followlinks: bool = False
@@ -917,11 +950,12 @@ class FSPath(URIPath):
     def open(
         self,
         mode: str = "r",
-        buffering=-1,
-        encoding=None,
-        errors=None,
-        newline=None,
-        closefd=True,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        closefd: bool = True,
+        atomic: bool = False,
         **kwargs,
     ) -> IO:
         if not isinstance(self.path_without_protocol, int) and (
@@ -932,7 +966,7 @@ class FSPath(URIPath):
                     self.path_without_protocol  # pyre-ignore[6]
                 )
             ).mkdir(parents=True, exist_ok=True)
-        return io.open(
+        fp = io.open(
             self.path_without_protocol,
             mode,
             buffering=buffering,
@@ -941,6 +975,9 @@ class FSPath(URIPath):
             newline=newline,
             closefd=closefd,
         )
+        if atomic and ("w" in mode or "x" in mode or "a" in mode):
+            return WrapAtomic(fp)
+        return fp
 
     @cached_property
     def parts(self) -> Tuple[str, ...]:

megfile/interfaces.py

@@ -1,6 +1,7 @@
 import os
 from abc import ABC, abstractmethod
 from io import IOBase, UnsupportedOperation
+from logging import getLogger as get_logger
 from typing import IO, AnyStr, Iterable, List, Optional
 
 from megfile.pathlike import (
@@ -31,6 +32,8 @@ __all__ = [
     "URIPath",
 ]
 
+_logger = get_logger(__name__)
+
 
 def fullname(o):
     klass = o.__class__
@@ -43,16 +46,28 @@ def fullname(o):
 # 1. Default value of closed is False
 # 2. closed is set to True when close() are called
 # 3. close() will only be called once
+# 4. atomic means the file-like object should not be closed automatically
+#    when an exception is raised in the context manager or when the object is
+#    garbage collected.
+# 5. atomic is False by default
 class Closable(ABC):
     @property
     def closed(self) -> bool:
         """Return True if the file-like object is closed."""
         return getattr(self, "__closed__", False)
 
+    @property
+    def atomic(self) -> bool:
+        """Return True if the file-like object is atomic."""
+        return getattr(self, "__atomic__", False)
+
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
+    def _abort(self) -> None:
+        pass
+
     def close(self) -> None:
         """Flush and close the file-like object.
 
@@ -66,6 +81,24 @@ class Closable(ABC):
         return self
 
     def __exit__(self, type, value, traceback) -> None:
+        if self.atomic and value is not None:
+            from megfile.errors import full_error_message
+
+            _logger.warning(
+                f"skip closing atomic file-like object: {self}, "
+                f"since error encountered: {full_error_message(value)}"
+            )
+            self._abort()
+            return
+        self.close()
+
+    def __del__(self):
+        if self.atomic:
+            _logger.warning(
+                f"skip closing atomic file-like object before deletion: {self}"
+            )
+            self._abort()
+            return
         self.close()
 
 

megfile/lib/joinpath.py

@@ -33,3 +33,16 @@ def uri_join(path: str, *other_paths: str) -> str:
 
     # Imp. 3
     # return '/'.join((path, *other_paths))
+
+
+def uri_norm(path: str) -> str:
+    parts = path.split("/")
+    new_parts = []
+    for part in parts:
+        if part == ".":
+            continue
+        if part == ".." and new_parts and new_parts[-1] != "..":
+            new_parts.pop()
+        else:
+            new_parts.append(part)
+    return "/".join(new_parts)

megfile/lib/s3_buffered_writer.py

@@ -53,11 +53,13 @@ class S3BufferedWriter(Writable[bytes]):
         max_buffer_size: int = WRITER_MAX_BUFFER_SIZE,
         max_workers: Optional[int] = None,
         profile_name: Optional[str] = None,
+        atomic: bool = False,
     ):
         self._bucket = bucket
         self._key = key
         self._client = s3_client
         self._profile_name = profile_name
+        self.__atomic__ = atomic
 
         # user maybe put block_size with 'numpy.uint64' type
         self._base_block_size = int(block_size)
@@ -213,6 +215,17 @@ class S3BufferedWriter(Writable[bytes]):
         if not self._is_global_executor:
             self._executor.shutdown()
 
+    def _abort(self):
+        _logger.debug("abort file: %r" % self.name)
+
+        if self._is_multipart:
+            with raise_s3_error(self.name):
+                self._client.abort_multipart_upload(
+                    Bucket=self._bucket, Key=self._key, UploadId=self._upload_id
+                )
+
+        self._shutdown()
+
     def _close(self):
         _logger.debug("close file: %r" % self.name)
 

megfile/lib/s3_limited_seekable_writer.py

@@ -33,6 +33,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
         max_buffer_size: int = WRITER_MAX_BUFFER_SIZE,
         max_workers: Optional[int] = None,
         profile_name: Optional[str] = None,
+        atomic: bool = False,
     ):
         super().__init__(
             bucket,
@@ -42,6 +43,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
             max_buffer_size=max_buffer_size,
             max_workers=max_workers,
             profile_name=profile_name,
+            atomic=atomic,
         )
 
         self._head_block_size = head_block_size or block_size

megfile/s3_path.py

@@ -230,7 +230,7 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str:
     config_endpoint_url = config.get("s3", {}).get("endpoint_url")
     config_endpoint_url = config_endpoint_url or config.get("endpoint_url")
     if config_endpoint_url:
-        warning_endpoint_url("~/.aws/config", config_endpoint_url)
+        warning_endpoint_url("~/.aws/config or ~/.aws/credentials", config_endpoint_url)
         return config_endpoint_url
     return endpoint_url
 
@@ -937,6 +937,7 @@ def s3_buffered_open(
     buffered: bool = False,
     share_cache_key: Optional[str] = None,
     cache_path: Optional[str] = None,
+    atomic: bool = False,
 ) -> IO:
     """Open an asynchronous prefetch reader, to support fast sequential read
 
@@ -1045,6 +1046,7 @@ def s3_buffered_open(
             block_size=block_size,
             max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
+            atomic=atomic,
         )
     else:
         if max_buffer_size is None:
@@ -1057,6 +1059,7 @@ def s3_buffered_open(
             block_size=block_size,
             max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
+            atomic=atomic,
         )
     if buffered or _is_pickle(writer):
         writer = io.BufferedWriter(writer)  # type: ignore

megfile/sftp2_path.py

@@ -14,6 +14,7 @@ from urllib.parse import urlsplit, urlunsplit
 import ssh2.session  # type: ignore
 import ssh2.sftp  # type: ignore
 from ssh2.exceptions import SFTPProtocolError  # type: ignore
+from ssh2.sftp_handle import SFTPAttributes  # type: ignore
 
 from megfile.config import SFTP_MAX_RETRY_TIMES
 from megfile.errors import SameFileError, _create_missing_ok_generator
@@ -884,7 +885,7 @@ class Sftp2Path(URIPath):
         if not self.is_symlink():
             raise OSError(f"Not a symlink: {self.path_with_protocol!r}")
         try:
-            path = self._client.readlink(self._real_path)
+            path = self._client.realpath(self._real_path)
             if not path:
                 raise OSError(f"Not a symlink: {self.path_with_protocol!r}")
             if not path.startswith("/"):
@@ -976,7 +977,9 @@ class Sftp2Path(URIPath):
 
     def chmod(self, mode: int, *, follow_symlinks: bool = True):
         """Change the file mode and permissions"""
-        return self._client.setstat(self._real_path, mode)
+        stat = SFTPAttributes()
+        stat.permissions = int(mode)
+        return self._client.setstat(self._real_path, stat)
 
     def absolute(self) -> "Sftp2Path":
         """Make the path absolute"""
@@ -1081,4 +1084,7 @@ class Sftp2Path(URIPath):
 
     def utime(self, atime: Union[float, int], mtime: Union[float, int]) -> None:
         """Set the access and modified times of the file"""
-        self._client.utime(self._real_path, (atime, mtime))
+        stat = SFTPAttributes()
+        stat.atime = int(atime)
+        stat.mtime = int(mtime)
+        self._client.setstat(self._real_path, stat)

megfile/sftp_path.py

@@ -1133,7 +1133,7 @@ class SftpPath(URIPath):
 
         src_stat = self.stat()
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
-        dst_path._client.chmod(dst_path._real_path, src_stat.st_mode)
+        dst_path.chmod(src_stat.st_mode)
 
     def sync(
         self,

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.2.4"
+VERSION = "4.2.5"