maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/lib/filesystem/_oss_lib/common.py

@@ -12,15 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
-import json
+import logging
 import os
+from typing import NamedTuple, Optional
+from urllib.parse import parse_qs, urlparse
 
 from ....utils import lazy_import
 from ..base import path_type, stringify_path
 
 oss2 = lazy_import("oss2", placeholder=True)
 
+logger = logging.getLogger(__name__)
+
 # OSS api time out
 _oss_time_out = 10
 
@@ -37,6 +40,8 @@ class OSSFileEntry:
         self._storage_options = storage_options
 
     def is_dir(self):
+        if self._path.endswith("/"):
+            self._is_dir = True
         if self._is_dir is None:
             self._is_dir = oss_isdir(self._path)
         return self._is_dir
@@ -63,43 +68,81 @@ class OSSFileEntry:
         return self._path
 
 
-def parse_osspath(path: path_type):
+class ParsedOSSPath(NamedTuple):
+    endpoint: str
+    bucket: str
+    key: str
+    access_key_id: Optional[str] = None
+    access_key_secret: Optional[str] = None
+    security_token: Optional[str] = None
+    scheme: str = None
+
+
+def parse_osspath(path: path_type, check_errors: bool = True) -> ParsedOSSPath:
     # Extract OSS configuration from the encoded URL.
     str_path = stringify_path(path)
-    parse_result = oss2.urlparse(str_path)
-    if parse_result.scheme != "oss":
+    parse_result = urlparse(str_path)
+    if check_errors and parse_result.scheme != "oss":
         raise ValueError(
             f"Except scheme oss, but got scheme: {parse_result.scheme}"
             f" in path: {str_path}"
         )
-    bucket = parse_result.hostname
-    if not (parse_result.username and parse_result.password):
-        raise RuntimeError(r"Please use build_oss_path to add OSS info")
-    param_dict = url_to_dict(parse_result.username)
-    access_key_id = param_dict["access_key_id"]
+    access_key_id = parse_result.username
     access_key_secret = parse_result.password
-    end_point = param_dict["end_point"]
+
+    if not parse_result.query:
+        sts_token = None
+    else:
+        sts_token = parse_qs(parse_result.query).get("security_token", [None])[0]
+
+    if check_errors and not (access_key_id and access_key_secret):
+        raise ValueError(r"No credentials provided")
+
     key = parse_result.path
     key = key[1:] if key.startswith("/") else key
-    return bucket, key, access_key_id, access_key_secret, end_point
+    if "/" not in key:
+        bucket, key = key, None
+        if check_errors:
+            raise ValueError("Need to use format bucket/key to separate bucket and key")
+    else:
+        bucket, key = key.split("/", 1)
+
+    endpoint = parse_result.hostname
+    if endpoint and parse_result.port:
+        endpoint += f":{parse_result.port}"
+    return ParsedOSSPath(
+        endpoint,
+        bucket,
+        key,
+        access_key_id,
+        access_key_secret,
+        sts_token,
+        parse_result.scheme,
+    )
 
 
-def _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point):
+def get_oss_bucket(parsed_path: ParsedOSSPath):
+    if parsed_path.security_token is not None:
+        auth = oss2.StsAuth(
+            parsed_path.access_key_id,
+            parsed_path.access_key_secret,
+            parsed_path.security_token,
+        )
+    else:
+        auth = oss2.Auth(parsed_path.access_key_id, parsed_path.access_key_secret)
     oss_bucket = oss2.Bucket(
-        auth=oss2.Auth(
-            access_key_id=access_key_id, access_key_secret=access_key_secret
-        ),
-        endpoint=end_point,
-        bucket_name=bucket,
+        auth=auth,
+        endpoint=parsed_path.endpoint,
+        bucket_name=parsed_path.bucket,
         connect_timeout=_oss_time_out,
     )
     return oss_bucket
 
 
 def oss_exists(path: path_type):
-    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(path)
-    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
-    return oss_bucket.object_exists(key) or oss_isdir(path)
+    parsed_path = parse_osspath(path)
+    oss_bucket = get_oss_bucket(parsed_path)
+    return oss_bucket.object_exists(parsed_path.key) or oss_isdir(path)
 
 
 def oss_isdir(path: path_type):
@@ -112,26 +155,71 @@ def oss_isdir(path: path_type):
     dirname = stringify_path(path)
     if not dirname.endswith("/"):
         dirname = dirname + "/"
-    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(dirname)
-    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    logger.info("Checking isdir for path %s", dirname)
+    parsed_path = parse_osspath(dirname)
+    oss_bucket = get_oss_bucket(parsed_path)
     isdir = False
-    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key, max_keys=2):
-        if obj.key == key:
+    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=parsed_path.key, max_keys=2):
+        if obj.key == parsed_path.key:
             continue
         isdir = True
         break
     return isdir
 
 
+def oss_delete(path: path_type):
+    """
+    Perform both key deletion and prefix deletion. Once no files
+    deleted in both scenarios, we can make assertion that the file
+    does not exist.
+    """
+    parsed_path = parse_osspath(path)
+    oss_bucket = get_oss_bucket(parsed_path)
+
+    try:
+        oss_bucket.delete_object(parsed_path.key)
+        return
+    except oss2.exceptions.NoSuchKey:
+        pass
+
+    is_missing = True
+    dir_key = parsed_path.key.rstrip("/") + "/"
+    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=dir_key):
+        try:
+            oss_bucket.delete_object(obj.key)
+            is_missing = False
+        except oss2.exceptions.NoSuchKey:
+            pass
+    if is_missing:
+        raise FileNotFoundError("No such file or directory: %s", path)
+
+
+def oss_copy_file(src_path: path_type, dest_path: path_type):
+    # todo implements copy of huge files
+    parsed_src_path = parse_osspath(src_path)
+    parsed_dest_path = parse_osspath(dest_path)
+    try:
+        if oss_isdir(src_path):
+            raise NotImplementedError("Copying directories not implemented yet")
+    except:
+        # fixme currently we cannot handle error with iterating files with STS token
+        logger.exception("Failed to judge if src is a directory")
+
+    oss_bucket = get_oss_bucket(parsed_dest_path)
+    oss_bucket.copy_object(
+        parsed_src_path.bucket, parsed_src_path.key, parsed_dest_path.key
+    )
+
+
 def oss_stat(path: path_type):
     path = stringify_path(path)
-    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(path)
-    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    parsed_path = parse_osspath(path)
+    oss_bucket = get_oss_bucket(parsed_path)
     if oss_isdir(path):
         stat = dict(name=path, size=0, modified_time=-1)
         stat["type"] = "directory"
     else:
-        meta = oss_bucket.get_object_meta(key)
+        meta = oss_bucket.get_object_meta(parsed_path.key)
         stat = dict(
             name=path,
             size=int(meta.headers["Content-Length"]),
@@ -145,11 +233,11 @@ def oss_scandir(dirname: path_type):
     dirname = stringify_path(dirname)
     if not dirname.endswith("/"):
         dirname = dirname + "/"
-    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(dirname)
-    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    parsed_path = parse_osspath(dirname)
+    oss_bucket = get_oss_bucket(parsed_path)
     dirname_set = set()
-    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key):
-        rel_path = obj.key[len(key) :]
+    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=parsed_path.key):
+        rel_path = obj.key[len(parsed_path.key) :]
         try:
             inside_dirname, inside_filename = rel_path.split("/", 1)
         except ValueError:
@@ -160,11 +248,11 @@ def oss_scandir(dirname: path_type):
                 continue
             dirname_set.add(inside_dirname)
             yield OSSFileEntry(
-                os.path.join(dirname, inside_dirname),
+                "/".join([dirname, inside_dirname]),
                 is_dir=True,
                 is_file=False,
                 stat={
-                    "name": os.path.join(dirname, inside_dirname),
+                    "name": "/".join([dirname, inside_dirname]),
                     "type": "directory",
                     "size": 0,
                     "modified_time": -1,
@@ -172,27 +260,13 @@ def oss_scandir(dirname: path_type):
             )
         else:
             yield OSSFileEntry(
-                os.path.join(dirname, inside_filename),
+                "/".join([dirname, inside_filename]),
                 is_dir=False,
                 is_file=True,
                 stat={
-                    "name": os.path.join(dirname, inside_filename),
+                    "name": "/".join([dirname, inside_filename]),
                     "type": "file",
                     "size": obj.size,
                     "modified_time": obj.last_modified,
                 },
             )
-
-
-def dict_to_url(param: dict):
-    # Encode the dictionary with url-safe-base64.
-    str_param = json.dumps(param)
-    url_param = base64.urlsafe_b64encode(bytes(str_param, encoding="utf8"))
-    return bytes.decode(url_param, encoding="utf8")
-
-
-def url_to_dict(url_param: str):
-    # Decode url-safe-base64 encoded string.
-    bytes_param = bytes(url_param, encoding="utf8")
-    str_param = bytes.decode(base64.urlsafe_b64decode(bytes_param), encoding="utf8")
-    return json.loads(str_param)

maxframe/lib/filesystem/_oss_lib/glob.py

@@ -104,7 +104,7 @@ def _glob0(dirname, basename, dironly):
         if oss_isdir(dirname):
             return [basename]
     else:
-        if oss_exists(os.path.join(dirname, basename)):
+        if oss_exists("/".join([dirname, basename])):
             return [basename]
     return []
 

maxframe/lib/filesystem/_oss_lib/handle.py

@@ -15,7 +15,7 @@
 from io import IOBase
 
 from ....utils import lazy_import
-from .common import oss_stat, parse_osspath
+from .common import get_oss_bucket, oss_stat, parse_osspath
 
 oss2 = lazy_import("oss2", placeholder=True)
 
@@ -23,20 +23,20 @@ oss2 = lazy_import("oss2", placeholder=True)
 class OSSIOBase(IOBase):
     def __init__(self, path, mode):
         self._path = path
-        (
-            self._bucket_name,
-            self._key_name,
-            self._access_key_id,
-            self._access_key_secret,
-            self._end_point,
-        ) = parse_osspath(self._path)
-        self._bucket = self._get_bucket()
+        self._parsed_path = parse_osspath(self._path)
+        self._bucket = get_oss_bucket(self._parsed_path)
         self._current_pos = 0
         self._size = None
         self._buffer = b""
         self._buffer_size = 1 * 1024
         self._mode = mode
 
+        if mode and mode.startswith("w"):
+            try:
+                self._bucket.delete_object(self._parsed_path.key)
+            except oss2.exceptions.NoSuchKey:
+                pass
+
     @property
     def mode(self):
         return self._mode
@@ -44,16 +44,6 @@ class OSSIOBase(IOBase):
     def fileno(self) -> int:
         raise AttributeError
 
-    def _get_bucket(self):
-        return oss2.Bucket(
-            auth=oss2.Auth(
-                access_key_id=self._access_key_id,
-                access_key_secret=self._access_key_secret,
-            ),
-            endpoint=self._end_point,
-            bucket_name=self._bucket_name,
-        )
-
     def _get_size(self):
         if self._size is None:
             self._size = int(oss_stat(self._path)["size"])
@@ -79,7 +69,7 @@ class OSSIOBase(IOBase):
         return self._current_pos
 
     def seekable(self):
-        return True
+        return "r" in self._mode
 
     def read(self, size=-1):
         """
@@ -97,12 +87,12 @@ class OSSIOBase(IOBase):
             return b""
         elif size < 0:
             obj = self._bucket.get_object(
-                self._key_name, byte_range=(self._current_pos, None)
+                self._parsed_path.key, byte_range=(self._current_pos, None)
             )
             self._current_pos = self._get_size()
         else:
             obj = self._bucket.get_object(
-                self._key_name,
+                self._parsed_path.key,
                 byte_range=(self._current_pos, self._current_pos + size - 1),
             )
             self._current_pos = self._current_pos + size
@@ -117,7 +107,7 @@ class OSSIOBase(IOBase):
                 self._get_size() - 1, self._current_pos + self._buffer_size - 1
             )
             buffer = self._bucket.get_object(
-                self._key_name, byte_range=(self._current_pos, read_to)
+                self._parsed_path.key, byte_range=(self._current_pos, read_to)
             ).read()
             if not buffer:
                 return 1
@@ -145,11 +135,17 @@ class OSSIOBase(IOBase):
                 break
         return bytes(res)
 
+    def write(self, block):
+        append_result = self._bucket.append_object(
+            self._parsed_path.key, self._current_pos, block
+        )
+        self._current_pos = append_result.next_position
+
     def readable(self):
-        return True
+        return "r" in self._mode
 
     def writable(self):
-        return False
+        return "w" in self._mode or "a" in self._mode
 
     def close(self):
         # already closed by oss

maxframe/lib/filesystem/base.py

@@ -247,7 +247,7 @@ class FileSystem(ABC):
     def parse_from_path(uri: str):
         parsed_uri = urlparse(uri)
         options = dict()
-        options["host"] = parsed_uri.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0]
+        options["host"] = parsed_uri.hostname
         if parsed_uri.port:
             options["port"] = parsed_uri.port
         if parsed_uri.username:

maxframe/lib/filesystem/core.py

@@ -49,7 +49,7 @@ def get_fs(path: path_type, storage_options: Dict = None) -> FileSystem:
 
     if scheme in _filesystems:
         file_system_type = _filesystems[scheme]
-        if scheme == "file" or scheme == "oss":
+        if scheme == "file":
             # local file systems are singletons.
             return file_system_type.get_instance()
         else:

maxframe/lib/filesystem/oss.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict, Iterator, List, Tuple
-from urllib import parse
+import enum
+import re
+from typing import Dict, Iterator, List, Tuple, Union
+from urllib.parse import urlencode
 
 from ...utils import implements, lazy_import
 from ._oss_lib import common as oc
@@ -22,18 +24,43 @@ from ._oss_lib.handle import OSSIOBase
 from .base import FileSystem, path_type
 
 oss2 = lazy_import("oss2", placeholder=True)
+_ip_regex = re.compile(r"^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})")
 
 _oss_time_out = 10
 
 
-class OSSFileSystem(FileSystem):
-    _instance = None
+class HostEnforceType(enum.Enum):
+    no_enforce = 0
+    force_internal = 1
+    force_external = 2
+
 
-    @classmethod
-    def get_instance(cls):
-        if cls._instance is None:
-            cls._instance = OSSFileSystem()
-        return cls._instance
+class OSSFileSystem(FileSystem):
+    def __init__(
+        self,
+        access_key_id: str = None,
+        access_key_secret: str = None,
+        security_token: str = None,
+        host_enforce_type: Union[HostEnforceType, str] = HostEnforceType.no_enforce,
+        **kw,
+    ):
+        self._access_key_id = access_key_id or kw.get("user")
+        self._access_key_secret = access_key_secret or kw.get("password")
+        self._security_token = security_token
+        self._host_enforce_type = (
+            host_enforce_type
+            if isinstance(host_enforce_type, HostEnforceType)
+            else getattr(HostEnforceType, host_enforce_type)
+        )
+
+    def _rewrite_path(self, path: str) -> str:
+        return build_oss_path(
+            path,
+            access_key_id=self._access_key_id,
+            access_key_secret=self._access_key_secret,
+            security_token=self._security_token,
+            host_enforce_type=self._host_enforce_type,
+        )
 
     @implements(FileSystem.cat)
     def cat(self, path: path_type):
@@ -46,39 +73,37 @@ class OSSFileSystem(FileSystem):
         if not file_entry.is_dir():
             raise OSError("ls for file is not supported")
         else:
-            bucket, key, access_key_id, access_key_secret, end_point = oc.parse_osspath(
-                path
-            )
-            oss_bucket = oss2.Bucket(
-                auth=oss2.Auth(
-                    access_key_id=access_key_id, access_key_secret=access_key_secret
-                ),
-                endpoint=end_point,
-                bucket_name=bucket,
-                connect_timeout=_oss_time_out,
-            )
-            for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key):
+            parsed_path = oc.parse_osspath(path)
+            oss_bucket = oc.get_oss_bucket(parsed_path)
+            for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=parsed_path.key):
                 if obj.key.endswith("/"):
                     continue
-                obj_path = rf"oss://{bucket}/{obj.key}"
+                obj_path = rf"{parsed_path.bucket}/{obj.key}"
                 file_list.append(
                     build_oss_path(
-                        obj_path, access_key_id, access_key_secret, end_point
+                        obj_path,
+                        parsed_path.endpoint,
+                        parsed_path.access_key_id,
+                        parsed_path.access_key_secret,
+                        parsed_path.security_token,
                     )
                 )
         return file_list
 
     @implements(FileSystem.delete)
     def delete(self, path: path_type, recursive: bool = False):
-        raise NotImplementedError
+        return oc.oss_delete(self._rewrite_path(path))
 
     @implements(FileSystem.rename)
     def rename(self, path: path_type, new_path: path_type):
-        raise NotImplementedError
+        # in OSS, you need to move file by copy and delete
+        path = self._rewrite_path(path)
+        oc.oss_copy_file(path, self._rewrite_path(new_path))
+        oc.oss_delete(path)
 
     @implements(FileSystem.stat)
     def stat(self, path: path_type) -> Dict:
-        ofe = oc.OSSFileEntry(path)
+        ofe = oc.OSSFileEntry(self._rewrite_path(path))
         return ofe.stat()
 
     @implements(FileSystem.mkdir)
@@ -87,12 +112,12 @@ class OSSFileSystem(FileSystem):
 
     @implements(FileSystem.isdir)
     def isdir(self, path: path_type) -> bool:
-        file_entry = oc.OSSFileEntry(path)
+        file_entry = oc.OSSFileEntry(self._rewrite_path(path))
         return file_entry.is_dir()
 
     @implements(FileSystem.isfile)
     def isfile(self, path: path_type) -> bool:
-        file_entry = oc.OSSFileEntry(path)
+        file_entry = oc.OSSFileEntry(self._rewrite_path(path))
         return file_entry.is_file()
 
     @implements(FileSystem._isfilestore)
@@ -101,11 +126,11 @@ class OSSFileSystem(FileSystem):
 
     @implements(FileSystem.exists)
     def exists(self, path: path_type):
-        return oc.oss_exists(path)
+        return oc.oss_exists(self._rewrite_path(path))
 
     @implements(FileSystem.open)
     def open(self, path: path_type, mode: str = "rb") -> OSSIOBase:
-        file_handle = OSSIOBase(path, mode)
+        file_handle = OSSIOBase(self._rewrite_path(path), mode)
         return file_handle
 
     @implements(FileSystem.walk)
@@ -114,10 +139,37 @@ class OSSFileSystem(FileSystem):
 
     @implements(FileSystem.glob)
     def glob(self, path: path_type, recursive: bool = False) -> List[path_type]:
-        return glob(path, recursive=recursive)
-
-
-def build_oss_path(path: path_type, access_key_id, access_key_secret, end_point):
+        return glob(self._rewrite_path(path), recursive=recursive)
+
+
+def _rewrite_internal_endpoint(
+    endpoint: str, host_enforce_type: HostEnforceType = HostEnforceType.no_enforce
+) -> str:
+    if (
+        not endpoint
+        or host_enforce_type == HostEnforceType.no_enforce
+        or _ip_regex.match(endpoint)
+    ):
+        return endpoint
+
+    ep_first, ep_rest = endpoint.split(".", 1)
+    host_with_internal = ep_first.endswith("-internal")
+    if host_enforce_type == HostEnforceType.force_external and host_with_internal:
+        return ep_first.replace("-internal", "") + "." + ep_rest
+    elif host_enforce_type == HostEnforceType.force_internal and not host_with_internal:
+        return ep_first + "-internal." + ep_rest
+    else:
+        return endpoint
+
+
+def build_oss_path(
+    path: path_type,
+    endpoint: str = None,
+    access_key_id: str = None,
+    access_key_secret: str = None,
+    security_token: str = None,
+    host_enforce_type: HostEnforceType = HostEnforceType.no_enforce,
+):
     """
     Returns a path with oss info.
     Used to register the access_key_id, access_key_secret and
@@ -127,16 +179,19 @@ def build_oss_path(path: path_type, access_key_id, access_key_secret, end_point)
     Parameters
     ----------
     path : path_type
-        The original oss url.
+        The original OSS url.
+
+    endpoint : str
+        The endpoint of OSS.
 
     access_key_id : str
-        The access key id of oss.
+        The access key id of OSS.
 
     access_key_secret : str
-        The access key secret of oss.
+        The access key secret of OSS.
 
-    end_point : str
-        The endpoint of oss.
+    security_token : str
+        The security token of OSS.
 
     Returns
     -------
@@ -146,12 +201,26 @@ def build_oss_path(path: path_type, access_key_id, access_key_secret, end_point)
     """
     if isinstance(path, (list, tuple)):
         path = path[0]
-    param_dict = {"access_key_id": access_key_id, "end_point": end_point}
-    id_endpoint = oc.dict_to_url(param_dict)
-    password = access_key_secret
-    parse_result = parse.urlparse(path)
-    new_path = (
-        f"{parse_result.scheme}://{id_endpoint}:{password}"
-        f"@{parse_result.netloc}{parse_result.path}"
+    parse_result = oc.parse_osspath(path, check_errors=False)
+    access_key_id = parse_result.access_key_id or access_key_id
+    access_key_secret = parse_result.access_key_secret or access_key_secret
+    security_token = parse_result.security_token or security_token
+
+    scheme = parse_result.scheme or "oss"
+    endpoint = _rewrite_internal_endpoint(
+        parse_result.endpoint or endpoint, host_enforce_type
     )
+
+    if access_key_id and access_key_secret:
+        creds = f"{access_key_id}:{access_key_secret}@"
+    else:
+        creds = ""
+
+    new_path = f"{scheme}://{creds}{endpoint}/{parse_result.bucket}"
+    if parse_result.key:
+        new_path += f"/{parse_result.key}"
+    if security_token:
+        new_path += f"?{urlencode(dict(security_token=security_token))}"
+    # reparse to check errors
+    oc.parse_osspath(new_path)
     return new_path