megfile 4.2.4__py3-none-any.whl → 5.0.0__py3-none-any.whl

megfile/interfaces.py

@@ -1,6 +1,7 @@
 import os
 from abc import ABC, abstractmethod
 from io import IOBase, UnsupportedOperation
+from logging import getLogger as get_logger
 from typing import IO, AnyStr, Iterable, List, Optional
 
 from megfile.pathlike import (
@@ -31,6 +32,8 @@ __all__ = [
     "URIPath",
 ]
 
+_logger = get_logger(__name__)
+
 
 def fullname(o):
     klass = o.__class__
@@ -43,16 +46,39 @@ def fullname(o):
 # 1. Default value of closed is False
 # 2. closed is set to True when close() are called
 # 3. close() will only be called once
+# 4. atomic means the file-like object should not be closed automatically
+#    when an exception is raised in the context manager or when the object is
+#    garbage collected.
+# 5. atomic is False by default
 class Closable(ABC):
     @property
     def closed(self) -> bool:
         """Return True if the file-like object is closed."""
         return getattr(self, "__closed__", False)
 
+    @property
+    def atomic(self) -> bool:
+        """Return True if the file-like object is atomic."""
+        return getattr(self, "__atomic__", False)
+
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
+    def _abort(self) -> None:
+        pass
+
+    def abort(self) -> bool:
+        """Abort the file-like object without saving.
+
+        This method has no effect if the file is already closed.
+        """
+        if not getattr(self, "__closed__", False):
+            self._abort()
+            setattr(self, "__closed__", True)
+            return True
+        return False
+
     def close(self) -> None:
         """Flush and close the file-like object.
 
@@ -66,6 +92,24 @@ class Closable(ABC):
         return self
 
     def __exit__(self, type, value, traceback) -> None:
+        if self.atomic and value is not None:
+            if self.abort():
+                from megfile.errors import full_error_message
+
+                _logger.warning(
+                    f"skip closing atomic file-like object: {self}, "
+                    f"since error encountered: {full_error_message(value)}"
+                )
+            return
+        self.close()
+
+    def __del__(self):
+        if self.atomic:
+            if self.abort():
+                _logger.warning(
+                    f"skip closing atomic file-like object before deletion: {self}"
+                )
+            return
         self.close()
 
 

megfile/lib/base_memory_handler.py

@@ -0,0 +1,92 @@
+import os
+from abc import ABC, abstractmethod
+from io import BytesIO, UnsupportedOperation
+from typing import Iterable, List, Optional
+
+from megfile.interfaces import Readable, Seekable, Writable
+
+
+class BaseMemoryHandler(Readable[bytes], Seekable, Writable[bytes], ABC):
+    def __init__(
+        self,
+        mode: str,
+    ):
+        self._mode = mode
+
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
+            raise ValueError("unacceptable mode: %r" % mode)
+
+        self._fileobj = BytesIO()
+        self._download_fileobj()
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @property
+    def mode(self) -> str:
+        return self._mode
+
+    def tell(self) -> int:
+        return self._fileobj.tell()
+
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        return self._fileobj.seek(offset, whence)
+
+    def readable(self) -> bool:
+        return self._mode[0] == "r" or self._mode[-1] == "+"
+
+    def read(self, size: Optional[int] = None) -> bytes:
+        if not self.readable():
+            raise UnsupportedOperation("not readable")
+        return self._fileobj.read(size)
+
+    def readline(self, size: Optional[int] = None) -> bytes:
+        if not self.readable():
+            raise UnsupportedOperation("not readable")
+        if size is None:
+            size = -1
+        return self._fileobj.readline(size)
+
+    def readlines(self, hint: Optional[int] = None) -> List[bytes]:
+        if not self.readable():
+            raise UnsupportedOperation("not readable")
+        if hint is None:
+            hint = -1
+        return self._fileobj.readlines(hint)
+
+    def writable(self) -> bool:
+        return self._mode[0] == "w" or self._mode[0] == "a" or self._mode[-1] == "+"
+
+    def flush(self):
+        self._fileobj.flush()
+
+    def write(self, data: bytes) -> int:
+        if not self.writable():
+            raise UnsupportedOperation("not writable")
+        if self._mode[0] == "a":
+            self.seek(0, os.SEEK_END)
+        return self._fileobj.write(data)
+
+    def writelines(self, lines: Iterable[bytes]):
+        if not self.writable():
+            raise UnsupportedOperation("not writable")
+        if self._mode[0] == "a":
+            self.seek(0, os.SEEK_END)
+        self._fileobj.writelines(lines)
+
+    @abstractmethod
+    def _download_fileobj(self):
+        pass
+
+    @abstractmethod
+    def _upload_fileobj(self):
+        pass
+
+    def _close(self, need_upload: bool = True):
+        if hasattr(self, "_fileobj"):
+            need_upload = need_upload and self.writable()
+            if need_upload:
+                self._upload_fileobj()
+            self._fileobj.close()

megfile/lib/glob.py

@@ -289,9 +289,9 @@ def get_non_glob_dir(glob: str):
     root_dir = []
     if glob.startswith("/"):
         root_dir.append("/")
-    elif "://" in glob:
-        protocol, glob = glob.split("://", 1)
-        root_dir.append(f"{protocol}://")
+    elif "//" in glob:
+        protocol_or_domain, glob = glob.rsplit("//", 1)
+        root_dir.append(f"{protocol_or_domain}//")
     for name in glob.split("/"):
         if has_magic(name):
             break

megfile/lib/http_prefetch_reader.py

@@ -85,29 +85,29 @@ class HttpPrefetchReader(BasePrefetchReader):
                         "Cookies": response.cookies,
                         "StatusCode": response.status_code,
                     }
-            else:
-                range_end = end
-                if self._content_size is not None:
-                    range_end = min(range_end, self._content_size - 1)
-                headers = {"Range": f"bytes={start}-{range_end}"}
-                with self._session.get(
-                    fspath(self._url), headers=headers, stream=True
-                ) as response:
-                    if len(response.content) != int(response.headers["Content-Length"]):
-                        raise HttpBodyIncompleteError(
-                            "The downloaded content is incomplete, "
-                            "expected size: %s, actual size: %d"
-                            % (
-                                response.headers["Content-Length"],
-                                len(response.content),
-                            )
+
+            range_end = end
+            if self._content_size is not None:
+                range_end = min(range_end, self._content_size - 1)
+            headers = {"Range": f"bytes={start}-{range_end}"}
+            with self._session.get(
+                fspath(self._url), headers=headers, stream=True
+            ) as response:
+                if len(response.content) != int(response.headers["Content-Length"]):
+                    raise HttpBodyIncompleteError(
+                        "The downloaded content is incomplete, "
+                        "expected size: %s, actual size: %d"
+                        % (
+                            response.headers["Content-Length"],
+                            len(response.content),
                         )
-                    return {
-                        "Body": BytesIO(response.content),
-                        "Headers": response.headers,
-                        "Cookies": response.cookies,
-                        "StatusCode": response.status_code,
-                    }
+                    )
+                return {
+                    "Body": BytesIO(response.content),
+                    "Headers": response.headers,
+                    "Cookies": response.cookies,
+                    "StatusCode": response.status_code,
+                }
 
         fetch_response = patch_method(
             fetch_response,

megfile/lib/joinpath.py

@@ -33,3 +33,16 @@ def uri_join(path: str, *other_paths: str) -> str:
 
     # Imp. 3
     # return '/'.join((path, *other_paths))
+
+
+def uri_norm(path: str) -> str:
+    parts = path.split("/")
+    new_parts = []
+    for part in parts:
+        if part == ".":
+            continue
+        if part == ".." and new_parts and new_parts[-1] != "..":
+            new_parts.pop()
+        else:
+            new_parts.append(part)
+    return "/".join(new_parts)

megfile/lib/s3_buffered_writer.py

@@ -53,11 +53,13 @@ class S3BufferedWriter(Writable[bytes]):
         max_buffer_size: int = WRITER_MAX_BUFFER_SIZE,
         max_workers: Optional[int] = None,
         profile_name: Optional[str] = None,
+        atomic: bool = False,
     ):
         self._bucket = bucket
         self._key = key
         self._client = s3_client
         self._profile_name = profile_name
+        self.__atomic__ = atomic
 
         # user maybe put block_size with 'numpy.uint64' type
         self._base_block_size = int(block_size)
@@ -213,6 +215,17 @@ class S3BufferedWriter(Writable[bytes]):
         if not self._is_global_executor:
             self._executor.shutdown()
 
+    def _abort(self):
+        _logger.debug("abort file: %r" % self.name)
+
+        if self._is_multipart:
+            with raise_s3_error(self.name):
+                self._client.abort_multipart_upload(
+                    Bucket=self._bucket, Key=self._key, UploadId=self._upload_id
+                )
+
+        self._shutdown()
+
     def _close(self):
         _logger.debug("close file: %r" % self.name)
 

megfile/lib/s3_limited_seekable_writer.py

@@ -33,6 +33,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
         max_buffer_size: int = WRITER_MAX_BUFFER_SIZE,
         max_workers: Optional[int] = None,
         profile_name: Optional[str] = None,
+        atomic: bool = False,
     ):
         super().__init__(
             bucket,
@@ -42,6 +43,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
             max_buffer_size=max_buffer_size,
             max_workers=max_workers,
             profile_name=profile_name,
+            atomic=atomic,
         )
 
         self._head_block_size = head_block_size or block_size

megfile/lib/s3_memory_handler.py

@@ -1,17 +1,16 @@
 import os
-from io import BytesIO, UnsupportedOperation
-from typing import Iterable, List, Optional
+from typing import Optional
 
 from megfile.errors import (
     S3ConfigError,
-    UnknownError,
-    raise_s3_error,
+    S3PermissionError,
+    S3UnknownError,
     translate_s3_error,
 )
-from megfile.interfaces import Readable, Seekable, Writable
+from megfile.lib.base_memory_handler import BaseMemoryHandler
 
 
-class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
+class S3MemoryHandler(BaseMemoryHandler):
     def __init__(
         self,
         bucket: str,
@@ -23,75 +22,14 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
     ):
         self._bucket = bucket
         self._key = key
-        self._mode = mode
         self._client = s3_client
         self._profile_name = profile_name
-
-        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
-        self._fileobj = BytesIO()
-        self._download_fileobj()
+        super().__init__(mode=mode)
 
     @property
     def name(self) -> str:
-        return "s3%s://%s/%s" % (
-            f"+{self._profile_name}" if self._profile_name else "",
-            self._bucket,
-            self._key,
-        )
-
-    @property
-    def mode(self) -> str:
-        return self._mode
-
-    def tell(self) -> int:
-        return self._fileobj.tell()
-
-    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        return self._fileobj.seek(offset, whence)
-
-    def readable(self) -> bool:
-        return self._mode[0] == "r" or self._mode[-1] == "+"
-
-    def read(self, size: Optional[int] = None) -> bytes:
-        if not self.readable():
-            raise UnsupportedOperation("not readable")
-        return self._fileobj.read(size)
-
-    def readline(self, size: Optional[int] = None) -> bytes:
-        if not self.readable():
-            raise UnsupportedOperation("not readable")
-        if size is None:
-            size = -1
-        return self._fileobj.readline(size)
-
-    def readlines(self, hint: Optional[int] = None) -> List[bytes]:
-        if not self.readable():
-            raise UnsupportedOperation("not readable")
-        if hint is None:
-            hint = -1
-        return self._fileobj.readlines(hint)
-
-    def writable(self) -> bool:
-        return self._mode[0] == "w" or self._mode[0] == "a" or self._mode[-1] == "+"
-
-    def flush(self):
-        self._fileobj.flush()
-
-    def write(self, data: bytes) -> int:
-        if not self.writable():
-            raise UnsupportedOperation("not writable")
-        if self._mode[0] == "a":
-            self.seek(0, os.SEEK_END)
-        return self._fileobj.write(data)
-
-    def writelines(self, lines: Iterable[bytes]):
-        if not self.writable():
-            raise UnsupportedOperation("not writable")
-        if self._mode[0] == "a":
-            self.seek(0, os.SEEK_END)
-        self._fileobj.writelines(lines)
+        protocol = f"s3+{self._profile_name}" if self._profile_name else "s3"
+        return f"{protocol}://{self._bucket}/{self._key}"
 
     def _translate_error(self, error: Exception):
         return translate_s3_error(error, self.name)
@@ -101,15 +39,14 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
             self._client.head_object(Bucket=self._bucket, Key=self._key)
         except Exception as error:
             error = self._translate_error(error)
-            if isinstance(error, (UnknownError, S3ConfigError)):
+            if isinstance(error, (S3UnknownError, S3ConfigError, S3PermissionError)):
                 raise error
             return False
         return True
 
     def _download_fileobj(self):
-        need_download = self._mode[0] == "r" or (
-            self._mode[0] == "a" and self._file_exists()
-        )
+        need_download = self._mode[0] == "r"
+        need_download = need_download or (self._mode[0] == "a" and self._file_exists())
         if not need_download:
             return
         # directly download to the file handle
@@ -126,11 +63,7 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
             return
         # directly upload from file handle
         self.seek(0, os.SEEK_SET)
-        with raise_s3_error(self.name):
+        try:
             self._client.upload_fileobj(self._fileobj, self._bucket, self._key)
-
-    def _close(self, need_upload: bool = True):
-        if hasattr(self, "_fileobj"):
-            if need_upload:
-                self._upload_fileobj()
-            self._fileobj.close()
+        except Exception as error:
+            raise self._translate_error(error)

megfile/lib/webdav_memory_handler.py

@@ -0,0 +1,83 @@
+import os
+
+from webdav3.client import Client as WebdavClient
+from webdav3.client import Urn, WebDavXmlUtils, wrap_connection_error
+from webdav3.exceptions import (
+    OptionNotValid,
+    RemoteResourceNotFound,
+)
+
+from megfile.lib.base_memory_handler import BaseMemoryHandler
+
+
+def _webdav_stat(client: WebdavClient, remote_path: str):
+    urn = Urn(remote_path)
+    response = client.execute_request(
+        action="info", path=urn.quote(), headers_ext=["Depth: 0"]
+    )
+    path = client.get_full_path(urn)
+    info = WebDavXmlUtils.parse_info_response(
+        response.content, path, client.webdav.hostname
+    )
+    info["is_dir"] = WebDavXmlUtils.parse_is_dir_response(
+        response.content, path, client.webdav.hostname
+    )
+    return info
+
+
+@wrap_connection_error
+def _webdav_download_from(client: WebdavClient, buff, remote_path):
+    urn = Urn(remote_path)
+    if client.is_dir(urn.path()):
+        raise OptionNotValid(name="remote_path", value=remote_path)
+
+    if not client.check(urn.path()):
+        raise RemoteResourceNotFound(urn.path())
+
+    response = client.execute_request(action="download", path=urn.quote())
+
+    for chunk in response.iter_content(chunk_size=client.chunk_size):
+        buff.write(chunk)
+
+
+class WebdavMemoryHandler(BaseMemoryHandler):
+    def __init__(
+        self,
+        remote_path: str,
+        mode: str,
+        *,
+        webdav_client: WebdavClient,
+        name: str,
+    ):
+        self._remote_path = remote_path
+        self._client = webdav_client
+        self._name = name
+        super().__init__(mode=mode)
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def _file_exists(self) -> bool:
+        try:
+            return not _webdav_stat(self._client, self._remote_path)["is_dir"]
+        except RemoteResourceNotFound:
+            return False
+
+    def _download_fileobj(self):
+        need_download = self._mode[0] == "r"
+        need_download = need_download or (self._mode[0] == "a" and self._file_exists())
+        if not need_download:
+            return
+        # directly download to the file handle
+        _webdav_download_from(self._client, self._fileobj, self._remote_path)
+        if self._mode[0] == "r":
+            self.seek(0, os.SEEK_SET)
+
+    def _upload_fileobj(self):
+        need_upload = self.writable()
+        if not need_upload:
+            return
+        # directly upload from file handle
+        self.seek(0, os.SEEK_SET)
+        self._client.upload_to(self._fileobj, self._remote_path)

megfile/lib/webdav_prefetch_reader.py

@@ -0,0 +1,115 @@
+from io import BytesIO
+from typing import Optional
+
+from webdav3.client import Client as WebdavClient
+from webdav3.client import Urn
+
+from megfile.config import (
+    READER_BLOCK_SIZE,
+    READER_MAX_BUFFER_SIZE,
+    WEBDAV_MAX_RETRY_TIMES,
+)
+from megfile.errors import (
+    HttpBodyIncompleteError,
+    http_should_retry,
+    patch_method,
+)
+from megfile.lib.base_prefetch_reader import BasePrefetchReader
+from megfile.lib.webdav_memory_handler import _webdav_stat
+
+DEFAULT_TIMEOUT = (60, 60 * 60 * 24)
+
+
+class WebdavPrefetchReader(BasePrefetchReader):
+    """
+    Reader to fast read the http content, service must support Accept-Ranges.
+
+    This will divide the file content into equal parts of block_size size, and will use
+    LRU to cache at most blocks in max_buffer_size memory.
+
+    open(), seek() and read() will trigger prefetch read.
+
+    The prefetch will cached block_forward blocks of data from offset position
+    (the position after reading if the called function is read).
+    """
+
+    def __init__(
+        self,
+        remote_path: str,
+        *,
+        client: Optional[WebdavClient] = None,
+        block_size: int = READER_BLOCK_SIZE,
+        max_buffer_size: int = READER_MAX_BUFFER_SIZE,
+        block_forward: Optional[int] = None,
+        max_retries: int = WEBDAV_MAX_RETRY_TIMES,
+        max_workers: Optional[int] = None,
+    ):
+        self._urn = Urn(remote_path)
+        self._remote_path = remote_path
+        self._client = client or WebdavClient({})
+
+        super().__init__(
+            block_size=block_size,
+            max_buffer_size=max_buffer_size,
+            block_forward=block_forward,
+            max_retries=max_retries,
+            max_workers=max_workers,
+        )
+
+    def _get_content_size(self) -> int:
+        info = _webdav_stat(self._client, self._remote_path)
+        return int(info.get("size") or 0)
+
+    @property
+    def name(self) -> str:
+        return self._remote_path
+
+    def _fetch_response(
+        self, start: Optional[int] = None, end: Optional[int] = None
+    ) -> dict:
+        def fetch_response() -> dict:
+            if start is None or end is None:
+                with self._client.execute_request(
+                    action="download", path=self._urn.quote()
+                ) as response:
+                    return {
+                        "Headers": response.headers,
+                        "Cookies": response.cookies,
+                        "StatusCode": response.status_code,
+                    }
+
+            range_end = end
+            if self._content_size is not None:
+                range_end = min(range_end, self._content_size - 1)
+            headers_ext = [f"Range: bytes={start}-{range_end}"]
+            with self._client.execute_request(
+                action="download", path=self._urn.quote(), headers_ext=headers_ext
+            ) as response:
+                headers = response.headers
+                if (
+                    "Content-Length" in headers
+                    and len(response.content) != int(headers["Content-Length"])
+                    and not headers.get("Content-Encoding")
+                ):
+                    raise HttpBodyIncompleteError(
+                        "The downloaded content is incomplete, "
+                        "expected size: %s, actual size: %d"
+                        % (
+                            headers["Content-Length"],
+                            len(response.content),
+                        )
+                    )
+                return {
+                    "Body": BytesIO(response.content),
+                    "Headers": response.headers,
+                    "Cookies": response.cookies,
+                    "StatusCode": response.status_code,
+                }
+
+        fetch_response = patch_method(
+            fetch_response,
+            max_retries=self._max_retries,
+            should_retry=http_should_retry,
+        )
+
+        return fetch_response()

megfile/pathlike.py

@@ -22,7 +22,6 @@ from megfile.lib.compat import PathLike as _PathLike
 from megfile.lib.compat import fspath
 from megfile.lib.fnmatch import _compile_pattern
 from megfile.lib.joinpath import uri_join
-from megfile.utils import classproperty
 
 Self = TypeVar("Self")
 
@@ -556,15 +555,15 @@ class BasePath:
         ) as f:
             return f.write(data)
 
-    @classproperty
+    @cached_property
     def drive(self) -> str:
         return ""
 
-    @classproperty
+    @cached_property
     def root(self) -> str:
         return self.protocol + "://"
 
-    @classproperty
+    @cached_property
     def anchor(self) -> str:
         return self.root  # pyre-ignore[7]