megfile 4.2.5__py3-none-any.whl → 5.0.1__py3-none-any.whl

megfile/webdav_path.py

@@ -2,6 +2,9 @@ import hashlib
 import io
 import os
 import re
+import shlex
+import subprocess
+import time
 from functools import cached_property
 from logging import getLogger as get_logger
 from typing import IO, BinaryIO, Callable, Iterable, Iterator, List, Optional, Tuple
@@ -10,27 +13,46 @@ from urllib.parse import quote, unquote, urlsplit, urlunsplit
 import dateutil.parser
 from webdav3.client import Client as WebdavClient
 from webdav3.client import WebDavXmlUtils
-from webdav3.exceptions import RemoteResourceNotFound, WebDavException
+from webdav3.exceptions import (
+    RemoteResourceNotFound,
+    ResponseErrorCode,
+    WebDavException,
+)
 from webdav3.urn import Urn
 
-from megfile.errors import SameFileError, _create_missing_ok_generator
+from megfile.config import (
+    READER_BLOCK_SIZE,
+    READER_MAX_BUFFER_SIZE,
+    WEBDAV_MAX_RETRY_TIMES,
+)
+from megfile.errors import (
+    SameFileError,
+    _create_missing_ok_generator,
+    http_should_retry,
+    patch_method,
+)
 from megfile.interfaces import (
     ContextIterator,
     FileEntry,
     PathLike,
-    Readable,
-    Seekable,
     StatResult,
-    Writable,
 )
 from megfile.lib.compare import is_same_file
 from megfile.lib.compat import fspath
 from megfile.lib.fnmatch import translate
 from megfile.lib.glob import has_magic
 from megfile.lib.joinpath import uri_join, uri_norm
+from megfile.lib.webdav_memory_handler import WebdavMemoryHandler, _webdav_stat
+from megfile.lib.webdav_prefetch_reader import WebdavPrefetchReader
 from megfile.pathlike import URIPath
 from megfile.smart_path import SmartPath
-from megfile.utils import calculate_md5, copyfileobj, get_binary_mode, thread_local
+from megfile.utils import (
+    _is_pickle,
+    binary_open,
+    calculate_md5,
+    copyfileobj,
+    thread_local,
+)
 
 _logger = get_logger(__name__)
 
@@ -42,6 +64,7 @@ __all__ = [
 WEBDAV_USERNAME = "WEBDAV_USERNAME"
 WEBDAV_PASSWORD = "WEBDAV_PASSWORD"
 WEBDAV_TOKEN = "WEBDAV_TOKEN"
+WEBDAV_TOKEN_COMMAND = "WEBDAV_TOKEN_COMMAND"
 WEBDAV_TIMEOUT = "WEBDAV_TIMEOUT"
 
 
@@ -55,7 +78,7 @@ def _make_stat(info: dict) -> StatResult:
     except Exception:
         mtime = 0.0
 
-    isdir = info.get("is_dir", False)
+    isdir = info.get("isdir", False)
 
     return StatResult(
         size=size,
@@ -81,6 +104,7 @@ def provide_connect_info(
     username: Optional[str] = None,
     password: Optional[str] = None,
     token: Optional[str] = None,
+    token_command: Optional[str] = None,
 ) -> dict:
     """Provide connection info for WebDAV client"""
     if not username:
@@ -89,6 +113,8 @@ def provide_connect_info(
         password = os.getenv(WEBDAV_PASSWORD)
     if not token:
         token = os.getenv(WEBDAV_TOKEN)
+    if not token_command:
+        token_command = os.getenv(WEBDAV_TOKEN_COMMAND)
 
     timeout = int(os.getenv(WEBDAV_TIMEOUT, "30"))
 
@@ -98,7 +124,9 @@ def provide_connect_info(
         "webdav_disable_check": True,
     }
 
-    if token:
+    if token_command:
+        options["webdav_token_command"] = token_command
+    elif token:
         options["webdav_token"] = token
     elif username and password:
         options["webdav_login"] = username
@@ -107,15 +135,83 @@ def provide_connect_info(
     return options
 
 
+def _patch_execute_request(
+    client: WebdavClient,
+    status_forcelist: Iterable[int] = (500, 502, 503, 504),
+    max_retries: int = WEBDAV_MAX_RETRY_TIMES,
+) -> WebdavClient:
+    def webdav_update_token_by_command():
+        cmds = shlex.split(client.webdav.token_command)
+        client.webdav.token_command_last_call = time.time()
+        client.webdav.token = subprocess.check_output(cmds).decode().strip()
+        _logger.debug("update webdav token by command: %s", client.webdav.token_command)
+
+    def webdav_should_retry(error: Exception) -> bool:
+        if http_should_retry(error):
+            return True
+        if (
+            isinstance(error, ResponseErrorCode)
+            and error.code == 401  # pytype: disable=attribute-error
+        ):
+            token_command = client.webdav.token_command  # pyre-ignore[16]
+            last_call = client.webdav.token_command_last_call  # pyre-ignore[16]
+            if token_command is not None and time.time() - last_call > 5:
+                webdav_update_token_by_command()
+                return True
+        return False
+
+    def after_callback(response, *args, **kwargs):
+        if response.status_code in status_forcelist:
+            response.raise_for_status()
+        return response
+
+    def before_callback(action, path, data=None, headers_ext=None):
+        # refresh token if needed
+        if client.webdav.token_command is not None and not client.webdav.token:
+            webdav_update_token_by_command()
+        _logger.debug(
+            "send http request: %s %r, with parameters: %s, headers: %s",
+            action,
+            path,
+            data,
+            headers_ext,
+        )
+
+    def retry_callback(error, action, path, data=None, headers_ext=None):
+        if data and hasattr(data, "seek"):
+            data.seek(0)
+        elif isinstance(data, Iterator):
+            _logger.warning("Can not retry http request with iterator data")
+            raise
+
+    client.execute_request = patch_method(
+        client.execute_request,
+        max_retries=max_retries,
+        should_retry=webdav_should_retry,
+        before_callback=before_callback,
+        after_callback=after_callback,
+        retry_callback=retry_callback,
+    )
+
+    return client
+
+
 def _get_webdav_client(
     hostname: str,
     username: Optional[str] = None,
     password: Optional[str] = None,
     token: Optional[str] = None,
+    token_command: Optional[str] = None,
 ) -> WebdavClient:
     """Get WebDAV client"""
-    options = provide_connect_info(hostname, username, password, token)
-    return WebdavClient(options)
+    options = provide_connect_info(hostname, username, password, token, token_command)
+    client = WebdavClient(options)
+    client.webdav.token_command = options.pop(  # pyre-ignore[16]
+        "webdav_token_command", None
+    )
+    client.webdav.token_command_last_call = 0  # pyre-ignore[16]
+    client = _patch_execute_request(client)
+    return client
 
 
 def get_webdav_client(
@@ -123,10 +219,11 @@ def get_webdav_client(
     username: Optional[str] = None,
     password: Optional[str] = None,
     token: Optional[str] = None,
+    token_command: Optional[str] = None,
 ) -> WebdavClient:
     """Get cached WebDAV client"""
     return thread_local(
-        f"webdav_client:{hostname},{username},{password},{token}",
+        f"webdav_client:{hostname},{username},{password},{token},{token_command}",
         _get_webdav_client,
         hostname,
         username,
@@ -160,23 +257,6 @@ def _webdav_scan_pairs(
         yield src_file_path, dst_file_path
 
 
-def _webdav_stat(client: WebdavClient, remote_path: str):
-    urn = Urn(remote_path)
-    client._check_remote_resource(remote_path, urn)
-
-    response = client.execute_request(
-        action="info", path=urn.quote(), headers_ext=["Depth: 0"]
-    )
-    path = client.get_full_path(urn)
-    info = WebDavXmlUtils.parse_info_response(
-        response.content, path, client.webdav.hostname
-    )
-    info["is_dir"] = WebDavXmlUtils.parse_is_dir_response(
-        response.content, path, client.webdav.hostname
-    )
-    return info
-
-
 def _webdav_scan(client: WebdavClient, remote_path: str) -> List[dict]:
     directory_urn = Urn(remote_path, directory=True)
     if directory_urn.path() != WebdavClient.root and not client.check(
@@ -208,112 +288,12 @@ def _webdav_split_magic(path: str) -> Tuple[str, str]:
     return path, ""
 
 
-class WebdavMemoryHandler(Readable[bytes], Seekable, Writable[bytes]):  # noqa: F821
-    def __init__(
-        self,
-        real_path: str,
-        mode: str,
-        *,
-        webdav_client: WebdavClient,
-        name: str,
-    ):
-        self._real_path = real_path
-        self._mode = mode
-        self._client = webdav_client
-        self._name = name
-
-        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
-        self._fileobj = io.BytesIO()
-        self._download_fileobj()
-
-    @property
-    def name(self) -> str:
-        return self._name
-
-    @property
-    def mode(self) -> str:
-        return self._mode
-
-    def tell(self) -> int:
-        return self._fileobj.tell()
-
-    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        return self._fileobj.seek(offset, whence)
-
-    def readable(self) -> bool:
-        return self._mode[0] == "r" or self._mode[-1] == "+"
-
-    def read(self, size: Optional[int] = None) -> bytes:
-        if not self.readable():
-            raise io.UnsupportedOperation("not readable")
-        return self._fileobj.read(size)
-
-    def readline(self, size: Optional[int] = None) -> bytes:
-        if not self.readable():
-            raise io.UnsupportedOperation("not readable")
-        if size is None:
-            size = -1
-        return self._fileobj.readline(size)
-
-    def readlines(self, hint: Optional[int] = None) -> List[bytes]:
-        if not self.readable():
-            raise io.UnsupportedOperation("not readable")
-        if hint is None:
-            hint = -1
-        return self._fileobj.readlines(hint)
-
-    def writable(self) -> bool:
-        return self._mode[0] == "w" or self._mode[0] == "a" or self._mode[-1] == "+"
-
-    def flush(self):
-        self._fileobj.flush()
-
-    def write(self, data: bytes) -> int:
-        if not self.writable():
-            raise io.UnsupportedOperation("not writable")
-        if self._mode[0] == "a":
-            self.seek(0, os.SEEK_END)
-        return self._fileobj.write(data)
-
-    def writelines(self, lines: Iterable[bytes]):
-        if not self.writable():
-            raise io.UnsupportedOperation("not writable")
-        if self._mode[0] == "a":
-            self.seek(0, os.SEEK_END)
-        self._fileobj.writelines(lines)
-
-    def _file_exists(self) -> bool:
-        try:
-            return not self._client.is_dir(self._real_path)
-        except RemoteResourceNotFound:
-            return False
-
-    def _download_fileobj(self):
-        need_download = self._mode[0] == "r" or (
-            self._mode[0] == "a" and self._file_exists()
-        )
-        if not need_download:
-            return
-        # directly download to the file handle
-        self._client.download_from(self._fileobj, self._real_path)
-        if self._mode[0] == "r":
-            self.seek(0, os.SEEK_SET)
-
-    def _upload_fileobj(self):
-        need_upload = self.writable()
-        if not need_upload:
-            return
-        # directly upload from file handle
-        self.seek(0, os.SEEK_SET)
-        self._client.upload_to(self._fileobj, self._real_path)
-
-    def _close(self, need_upload: bool = True):
-        if hasattr(self, "_fileobj"):
-            if need_upload:
-                self._upload_fileobj()
-            self._fileobj.close()
+def _webdav_check_accept_ranges(client: WebdavClient, remote_path: str):
+    urn = Urn(remote_path)
+    response = client.execute_request(action="download", path=urn.quote())
+    response.close()
+    headers = response.headers
+    return headers.get("Accept-Ranges") == "bytes"
 
 
 @SmartPath.register
@@ -346,16 +326,15 @@ class WebdavPath(URIPath):
         if parts.port:
             self._hostname += f":{parts.port}"
 
-        self._real_path = unquote(parts.path) if parts.path else "/"
+        self._remote_path = unquote(parts.path) if parts.path else "/"
 
     @cached_property
     def parts(self) -> Tuple[str, ...]:
         """A tuple giving access to the path's various components"""
-        new_parts = self._urlsplit_parts._replace(path="/")
-        parts: List[str] = [urlunsplit(new_parts)]  # pyre-ignore[9]
+        parts = [urlunsplit(self._urlsplit_parts._replace(path=""))]
         path = self._urlsplit_parts.path.lstrip("/")
         if path != "":
-            parts.extend(unquote(path).split("/"))
+            parts.extend(path.split("/"))
         return tuple(parts)
 
     @property
@@ -385,7 +364,7 @@ class WebdavPath(URIPath):
         :returns: True if the path exists, else False
         """
         try:
-            _webdav_stat(self._client, self._real_path)
+            _webdav_stat(self._client, self._remote_path)
             return True
         except RemoteResourceNotFound:
             return False
@@ -436,11 +415,11 @@ class WebdavPath(URIPath):
             raise FileNotFoundError
         :returns: An iterator contains tuples of path and file stat
         """
-        remote_path = self._real_path
+        remote_path = self._remote_path
         if pattern:
             remote_path = os.path.join(remote_path, pattern)
         remote_path, pattern = _webdav_split_magic(remote_path)
-        root = os.path.relpath(remote_path, self._real_path)
+        root = os.path.relpath(remote_path, self._remote_path)
         root = uri_join(self.path_with_protocol, root)
         root = uri_norm(root)
         pattern = re.compile(translate(pattern))
@@ -479,7 +458,7 @@ class WebdavPath(URIPath):
         :returns: True if the path is a directory, else False
         """
         try:
-            return _webdav_stat(self._client, self._real_path)["is_dir"]
+            return _webdav_stat(self._client, self._remote_path)["isdir"]
         except RemoteResourceNotFound:
             return False
 
@@ -491,7 +470,7 @@ class WebdavPath(URIPath):
         :returns: True if the path is a file, else False
         """
         try:
-            return not _webdav_stat(self._client, self._real_path)["is_dir"]
+            return not _webdav_stat(self._client, self._remote_path)["isdir"]
         except RemoteResourceNotFound:
             return False
 
@@ -550,7 +529,7 @@ class WebdavPath(URIPath):
                 parent_path_object.mkdir(mode=mode, parents=False, exist_ok=True)
 
         try:
-            self._client.mkdir(self._real_path)
+            self._client.mkdir(self._remote_path)
         except WebDavException:
             # Catch exception when mkdir concurrently
             if not self.exists():
@@ -590,7 +569,9 @@ class WebdavPath(URIPath):
         if self._is_same_backend(dst_path):
             if overwrite:
                 dst_path.remove(missing_ok=True)
-            self._client.move(self._real_path, dst_path._real_path, overwrite=overwrite)
+            self._client.move(
+                self._remote_path, dst_path._remote_path, overwrite=overwrite
+            )
         else:
             if self.is_dir():
                 for file_entry in self.scandir():
@@ -626,7 +607,7 @@ class WebdavPath(URIPath):
         if missing_ok and not self.exists():
             return
         try:
-            self._client.clean(self._real_path)
+            self._client.clean(self._remote_path)
         except RemoteResourceNotFound:
             if not missing_ok:
                 raise FileNotFoundError(f"No such file: '{self.path_with_protocol}'")
@@ -665,8 +646,8 @@ class WebdavPath(URIPath):
                 )
                 return
 
-            for info in _webdav_scan(self._client, self._real_path):
-                entry = _make_entry(info, self._real_path, self.path_with_protocol)
+            for info in _webdav_scan(self._client, self._remote_path):
+                entry = _make_entry(info, self._remote_path, self.path_with_protocol)
                 if entry.is_dir():
                     continue
                 yield entry
@@ -691,8 +672,8 @@ class WebdavPath(URIPath):
             raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
 
         def create_generator():
-            for info in _webdav_scandir(self._client, self._real_path):
-                yield _make_entry(info, self._real_path, self.path_with_protocol)
+            for info in _webdav_scandir(self._client, self._remote_path):
+                yield _make_entry(info, self._remote_path, self.path_with_protocol)
 
         return ContextIterator(create_generator())
 
@@ -703,7 +684,7 @@ class WebdavPath(URIPath):
         :returns: StatResult
         """
         try:
-            info = _webdav_stat(self._client, self._real_path)
+            info = _webdav_stat(self._client, self._remote_path)
             return _make_stat(info)
         except RemoteResourceNotFound:
             raise FileNotFoundError(f"No such file: '{self.path_with_protocol}'")
@@ -717,7 +698,7 @@ class WebdavPath(URIPath):
         if missing_ok and not self.exists():
             return
         try:
-            self._client.clean(self._real_path)
+            self._client.clean(self._remote_path)
         except RemoteResourceNotFound:
             if not missing_ok:
                 raise FileNotFoundError(f"No such file: '{self.path_with_protocol}'")
@@ -736,7 +717,7 @@ class WebdavPath(URIPath):
         if self.is_file():
             return
 
-        stack = [self._real_path]
+        stack = [self._remote_path]
         while stack:
             root = stack.pop()
             dirs, files = [], []
@@ -810,13 +791,15 @@ class WebdavPath(URIPath):
         with self.open(mode="wb") as output:
             output.write(file_object.read())
 
+    @binary_open
     def open(
         self,
-        mode: str = "r",
+        mode: str = "rb",
         *,
-        buffering=-1,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
+        max_workers: Optional[int] = None,
+        max_buffer_size: int = READER_MAX_BUFFER_SIZE,
+        block_forward: Optional[int] = None,
+        block_size: int = READER_BLOCK_SIZE,
         **kwargs,
     ) -> IO:
         """Open a file on the path.
@@ -837,15 +820,27 @@ class WebdavPath(URIPath):
         elif not self.exists():
             raise FileNotFoundError("No such file: %r" % self.path_with_protocol)
 
-        buffer = WebdavMemoryHandler(
-            self._real_path,
-            get_binary_mode(mode),
+        if mode == "rb":
+            if _webdav_check_accept_ranges(self._client, self._remote_path):
+                reader = WebdavPrefetchReader(
+                    self._remote_path,
+                    client=self._client,
+                    block_size=block_size,
+                    max_buffer_size=max_buffer_size,
+                    block_forward=block_forward,
+                    max_retries=WEBDAV_MAX_RETRY_TIMES,
+                    max_workers=max_workers,
+                )
+                if _is_pickle(reader):
+                    reader = io.BufferedReader(reader)  # type: ignore
+                return reader
+
+        return WebdavMemoryHandler(
+            self._remote_path,
+            mode,
             webdav_client=self._client,
             name=self.path_with_protocol,
         )
-        if "b" not in mode:
-            return io.TextIOWrapper(buffer, encoding=encoding, errors=errors)
-        return buffer
 
     def chmod(self, mode: int, *, follow_symlinks: bool = True):
         """
@@ -870,7 +865,7 @@ class WebdavPath(URIPath):
         """
         if len(self.listdir()) > 0:
             raise OSError(f"Directory not empty: '{self.path_with_protocol}'")
-        self._client.clean(self._real_path)
+        self._client.clean(self._remote_path)
 
     def copy(
         self,
@@ -902,11 +897,11 @@ class WebdavPath(URIPath):
         dst_path = self.from_path(dst_path)
 
         if self._is_same_backend(dst_path):
-            if self._real_path == dst_path._real_path:
+            if self._remote_path == dst_path._remote_path:
                 raise SameFileError(
                     f"'{self.path}' and '{dst_path.path}' are the same file"
                 )
-            self._client.copy(self._real_path, dst_path._real_path)
+            self._client.copy(self._remote_path, dst_path._remote_path)
             if callback:
                 callback(self.stat().size)
         else:

{megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.2.5
+Version: 5.0.1
 Summary: Megvii file operation library
-Author-email: megvii <megfile@megvii.com>
+Author-email: megvii-reng <megvii-reng@googlegroups.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -12,12 +12,12 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: LICENSE.pyre
@@ -51,21 +51,16 @@ megfile - Megvii FILE library
 
 * Docs: http://megvii-research.github.io/megfile
 
-`megfile` provides a silky operation experience with different backends (currently including local file system and s3), which enable you to focus more on the logic of your own project instead of the question of "Which backend is used for this file?"
+`megfile` provides a silky operation experience with different backends, which enable you to focus more on the logic of your own project instead of the question of "Which backend is used for this file?"
 
-`megfile` provides:
+## Why megfile
 
-* Almost unified file system operation experience. Target path can be easily moved from local file system to s3.
-* Complete boundary case handling. Even the most difficult (or even you can't even think of) boundary conditions, `megfile` can help you easily handle it.
-* Perfect type hints and built-in documentation. You can enjoy the IDE's auto-completion and static checking.
-* Semantic version and upgrade guide, which allows you enjoy the latest features easily.
-
-`megfile`'s advantages are:
-
-* `smart_open` can open resources that use various protocols. Especially, reader / writer of s3 in `megfile` is implemented with multi-thread, which is faster than known competitors.
-* `smart_glob` is available on majority protocols. And it supports zsh extended pattern syntax of `[]`, e.g. `s3://bucket/video.{mp4,avi}`.
-* All-inclusive functions like `smart_exists` / `smart_stat` / `smart_sync`. If you don't find the functions you want, [submit an issue](https://github.com/megvii-research/megfile/issues).
-* Compatible with `pathlib.Path` interface, referring to `SmartPath` and other protocol classes like `S3Path`.
+* Same interfaces as the python standard library, low learning curve
+* Faster file read and write operations
+* Excellent error retry mechanism to help you handle network issues
+* Supports popular protocols, even making it easy to use the same protocol with different endpoints
+* Stable and secure, with CI coverage over 95%, used by multiple industry giants
+* Perfect type hints and built-in documentation. You can enjoy the IDE's auto-completion and static checking
 
 ## Support Protocols
 - fs(local filesystem)
@@ -73,11 +68,15 @@ megfile - Megvii FILE library
 - sftp
 - http
 - stdio
-- hdfs: `pip install 'megfile[hdfs]'`
+- hdfs: `pip3 install 'megfile[hdfs]'`
+- webdav: `pip3 install 'megfile[webdav]'`
 
 ## Quick Start
 
-Path string in `megfile` almost is `protocol://path/to/file`, for example `s3://bucketA/key`. But sftp path is a little different, format is `sftp://[username[:password]@]hostname[:port]//absolute_file_path`. More details see [path format document](https://megvii-research.github.io/megfile/path_format.html).
+The interfaces of `megfile` correspond to those in the Python standard library. For example, `open` -> `smart_open` and `pathlib.Path` -> `SmartPath`. You only need to [configure the protocol settings](https://megvii-research.github.io/megfile/configuration.html) and provide the path in the corresponding format to use them conveniently.
+
+Path string in `megfile` almost is `protocol://path/to/file`, for example `s3://bucketA/key`. More details see [path format document](https://megvii-research.github.io/megfile/path_format.html).
+
 Here's an example of writing a file to s3 / fs, syncing to local, reading and finally deleting it.
 
 ### Functional Interface
@@ -106,7 +105,7 @@ smart_glob('s3://playground/megfile-?.{mp4,avi}')
 
 ### SmartPath Interface
 
-`SmartPath` has a similar interface with pathlib.Path.
+`SmartPath` has a similar interface with `pathlib.Path`.
 
 ```python
 from megfile.smart_path import SmartPath
@@ -138,28 +137,15 @@ $ megfile cp s3://playground/megfile-test /tmp/playground/megfile-test
 
 ```bash
 pip3 install megfile
-```
 
-You can specify megfile version as well
-```bash
-pip3 install "megfile~=0.0"
-```
+# for cli support
+pip3 install 'megfile[cli]'
 
-### Build from Source
+# for hdfs support
+pip3 install 'megfile[hdfs]'
 
-megfile can be installed from source
-```bash
-git clone git@github.com:megvii-research/megfile.git
-cd megfile
-pip3 install -U .
-```
-
-### Development Environment
-
-```bash
-git clone git@github.com:megvii-research/megfile.git
-cd megfile
-pip3 install -r requirements.txt -r requirements-dev.txt
+# for webdav support
+pip3 install 'megfile[webdav]'
 ```
 
 ## Configuration
@@ -218,6 +204,8 @@ You can get the configuration from `~/.config/megfile/aliases.conf`, like:
 protocol = s3+tos
 ```
 
+You can use alias in path, like `tos://bucket/key`, the same as `s3+tos://bucket/key`.
+
 ## Benchmark
 [![10GiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10GiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 [![10MiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10MiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)