maxframe 0.1.0b5__cp39-cp39-macosx_11_0_arm64.whl

maxframe/lib/filesystem/_oss_lib/common.py

@@ -0,0 +1,198 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import json
+import os
+
+from ....utils import lazy_import
+from ..base import path_type, stringify_path
+
+oss2 = lazy_import("oss2", placeholder=True)
+
+# OSS api time out
+_oss_time_out = 10
+
+
+class OSSFileEntry:
+    def __init__(
+        self, path, *, is_dir=None, is_file=None, stat=None, storage_options=None
+    ):
+        self._path = path
+        self._name = os.path.basename(path)
+        self._is_file = is_file
+        self._is_dir = is_dir
+        self._stat = stat
+        self._storage_options = storage_options
+
+    def is_dir(self):
+        if self._is_dir is None:
+            self._is_dir = oss_isdir(self._path)
+        return self._is_dir
+
+    def is_file(self):
+        if self._is_file is None:
+            if self.is_dir() or not oss_exists(self._path):
+                self._is_file = False
+            else:
+                self._is_file = True
+        return self._is_file
+
+    def stat(self):
+        if self._stat is None:
+            self._stat = oss_stat(self._path)
+        return self._stat
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def path(self):
+        return self._path
+
+
+def parse_osspath(path: path_type):
+    # Extract OSS configuration from the encoded URL.
+    str_path = stringify_path(path)
+    parse_result = oss2.urlparse(str_path)
+    if parse_result.scheme != "oss":
+        raise ValueError(
+            f"Except scheme oss, but got scheme: {parse_result.scheme}"
+            f" in path: {str_path}"
+        )
+    bucket = parse_result.hostname
+    if not (parse_result.username and parse_result.password):
+        raise RuntimeError(r"Please use build_oss_path to add OSS info")
+    param_dict = url_to_dict(parse_result.username)
+    access_key_id = param_dict["access_key_id"]
+    access_key_secret = parse_result.password
+    end_point = param_dict["end_point"]
+    key = parse_result.path
+    key = key[1:] if key.startswith("/") else key
+    return bucket, key, access_key_id, access_key_secret, end_point
+
+
+def _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point):
+    oss_bucket = oss2.Bucket(
+        auth=oss2.Auth(
+            access_key_id=access_key_id, access_key_secret=access_key_secret
+        ),
+        endpoint=end_point,
+        bucket_name=bucket,
+        connect_timeout=_oss_time_out,
+    )
+    return oss_bucket
+
+
+def oss_exists(path: path_type):
+    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(path)
+    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    return oss_bucket.object_exists(key) or oss_isdir(path)
+
+
+def oss_isdir(path: path_type):
+    """
+    OSS has no concept of directories, but we define
+    a ossurl is dir, When there is at least one object
+    at the ossurl that is the prefix(end with char "/"),
+    it is considered as a directory.
+    """
+    dirname = stringify_path(path)
+    if not dirname.endswith("/"):
+        dirname = dirname + "/"
+    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(dirname)
+    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    isdir = False
+    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key, max_keys=2):
+        if obj.key == key:
+            continue
+        isdir = True
+        break
+    return isdir
+
+
+def oss_stat(path: path_type):
+    path = stringify_path(path)
+    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(path)
+    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    if oss_isdir(path):
+        stat = dict(name=path, size=0, modified_time=-1)
+        stat["type"] = "directory"
+    else:
+        meta = oss_bucket.get_object_meta(key)
+        stat = dict(
+            name=path,
+            size=int(meta.headers["Content-Length"]),
+            modified_time=meta.headers["Last-Modified"],
+        )
+        stat["type"] = "file"
+    return stat
+
+
+def oss_scandir(dirname: path_type):
+    dirname = stringify_path(dirname)
+    if not dirname.endswith("/"):
+        dirname = dirname + "/"
+    bucket, key, access_key_id, access_key_secret, end_point = parse_osspath(dirname)
+    oss_bucket = _get_oss_bucket(bucket, access_key_id, access_key_secret, end_point)
+    dirname_set = set()
+    for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key):
+        rel_path = obj.key[len(key) :]
+        try:
+            inside_dirname, inside_filename = rel_path.split("/", 1)
+        except ValueError:
+            inside_dirname = None
+            inside_filename = rel_path
+        if inside_dirname is not None:
+            if inside_dirname in dirname_set:
+                continue
+            dirname_set.add(inside_dirname)
+            yield OSSFileEntry(
+                os.path.join(dirname, inside_dirname),
+                is_dir=True,
+                is_file=False,
+                stat={
+                    "name": os.path.join(dirname, inside_dirname),
+                    "type": "directory",
+                    "size": 0,
+                    "modified_time": -1,
+                },
+            )
+        else:
+            yield OSSFileEntry(
+                os.path.join(dirname, inside_filename),
+                is_dir=False,
+                is_file=True,
+                stat={
+                    "name": os.path.join(dirname, inside_filename),
+                    "type": "file",
+                    "size": obj.size,
+                    "modified_time": obj.last_modified,
+                },
+            )
+
+
+def dict_to_url(param: dict):
+    # Encode the dictionary with url-safe-base64.
+    str_param = json.dumps(param)
+    url_param = base64.urlsafe_b64encode(bytes(str_param, encoding="utf8"))
+    return bytes.decode(url_param, encoding="utf8")
+
+
+def url_to_dict(url_param: str):
+    # Decode url-safe-base64 encoded string.
+    bytes_param = bytes(url_param, encoding="utf8")
+    str_param = bytes.decode(base64.urlsafe_b64decode(bytes_param), encoding="utf8")
+    return json.loads(str_param)

maxframe/lib/filesystem/_oss_lib/glob.py

@@ -0,0 +1,147 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Filename globbing utility, modified from python glob.
+
+obviously，this implementation is not optimal, it will cause too many
+oss requests. Lately, We can then convert the glob expression into
+a regular expression, and then match the oss key list.
+But before that, we need to figure out how to deal with magic char
+in oss key, such like oss glob: oss://bucket/[key]/*, the key
+oss://bucket/[key]/a exactly exists.
+
+Notes:
+    OSS need a bucket to specify the file or dir, the "**" pattern is
+    not supported. So _isrecursive(pattern) is removed.
+"""
+
+import fnmatch
+import os
+import re
+
+from .common import oss_exists, oss_isdir, oss_scandir
+
+__all__ = ["glob", "iglob", "escape"]
+
+
+def glob(pathname, *, recursive=False):
+    """Return a list of paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    """
+    return list(iglob(pathname, recursive=recursive))
+
+
+def iglob(pathname, *, recursive=False):
+    """Return an iterator which yields the paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards like
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    """
+    it = _iglob(pathname, recursive, False)
+    return it
+
+
+def _iglob(pathname, recursive, dironly):
+    dirname, basename = os.path.split(pathname)
+    if not has_magic(pathname):
+        assert not dironly
+        if basename:
+            if oss_exists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if oss_isdir(dirname):
+                yield pathname
+        return
+    # dirname will not be None in oss path.
+    #  Prevent an infinite recursion if a drive or UNC path
+    # contains magic characters (i.e. r'\\?\C:').
+    if dirname != pathname and has_magic(dirname):
+        dirs = _iglob(dirname, recursive, True)
+    else:
+        dirs = [dirname]
+    if has_magic(basename):
+        glob_in_dir = _glob1
+    else:
+        glob_in_dir = _glob0
+    for dirname in dirs:
+        for name in glob_in_dir(dirname, basename, dironly):
+            yield os.path.join(dirname, name)
+
+
+# These 2 helper functions non-recursively glob inside a literal directory.
+# They return a list of basenames.  _glob1 accepts a pattern while _glob0
+# takes a literal basename (so it only has to check for its existence).
+
+
+def _glob1(dirname, pattern, dironly):
+    names = list(_iterdir(dirname, dironly))
+    if not _ishidden(pattern):
+        names = (x for x in names if not _ishidden(x))
+    return fnmatch.filter(names, pattern)
+
+
+def _glob0(dirname, basename, dironly):
+    if not basename:
+        # `os.path.split()` returns an empty basename for paths ending with a
+        # directory separator.  'q*x/' should match only directories.
+        if oss_isdir(dirname):
+            return [basename]
+    else:
+        if oss_exists(os.path.join(dirname, basename)):
+            return [basename]
+    return []
+
+
+# If dironly is false, yields all file names inside a directory.
+# If dironly is true, yields only directory names.
+# An oss path must contain a dirname.
+def _iterdir(dirname, dironly):
+    for entry in oss_scandir(dirname):
+        if not dironly or entry.is_dir():
+            yield entry.name
+    return
+
+
+magic_check = re.compile("([*?[])")
+magic_check_bytes = re.compile(b"([*?[])")
+
+
+def has_magic(s):
+    if isinstance(s, bytes):
+        match = magic_check_bytes.search(s)
+    else:
+        match = magic_check.search(s)
+    return match is not None
+
+
+def _ishidden(path):
+    return False
+
+
+def escape(pathname):
+    """Escape all special characters."""
+    # Escaping is done by wrapping any of "*?[" between square brackets.
+    # Metacharacters do not work in the drive part and shouldn't be escaped.
+    drive, pathname = os.path.splitdrive(pathname)
+    if isinstance(pathname, bytes):
+        pathname = magic_check_bytes.sub(rb"[\1]", pathname)
+    else:
+        pathname = magic_check.sub(r"[\1]", pathname)
+    return drive + pathname

maxframe/lib/filesystem/_oss_lib/handle.py

@@ -0,0 +1,156 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from io import IOBase
+
+from ....utils import lazy_import
+from .common import oss_stat, parse_osspath
+
+oss2 = lazy_import("oss2", placeholder=True)
+
+
+class OSSIOBase(IOBase):
+    def __init__(self, path, mode):
+        self._path = path
+        (
+            self._bucket_name,
+            self._key_name,
+            self._access_key_id,
+            self._access_key_secret,
+            self._end_point,
+        ) = parse_osspath(self._path)
+        self._bucket = self._get_bucket()
+        self._current_pos = 0
+        self._size = None
+        self._buffer = b""
+        self._buffer_size = 1 * 1024
+        self._mode = mode
+
+    @property
+    def mode(self):
+        return self._mode
+
+    def fileno(self) -> int:
+        raise AttributeError
+
+    def _get_bucket(self):
+        return oss2.Bucket(
+            auth=oss2.Auth(
+                access_key_id=self._access_key_id,
+                access_key_secret=self._access_key_secret,
+            ),
+            endpoint=self._end_point,
+            bucket_name=self._bucket_name,
+        )
+
+    def _get_size(self):
+        if self._size is None:
+            self._size = int(oss_stat(self._path)["size"])
+        return self._size
+
+    def seek(self, pos, whence=0):
+        if whence == 0:
+            if pos < 0:
+                raise OSError("Invalid argument")
+            self._current_pos = pos
+        elif whence == 2:
+            self._current_pos = self._get_size() + pos
+        elif whence == 1:
+            check_pos = self._current_pos + pos
+            if check_pos < 0:
+                raise OSError("Invalid argument")
+            else:
+                self._current_pos = self._current_pos + pos
+        else:
+            raise ValueError('Parameter "whence" should be 0 or 1 or 2')
+        if pos > 0 and self._current_pos > self._get_size() - 1:
+            self._current_pos = self._get_size()
+        return self._current_pos
+
+    def seekable(self):
+        return True
+
+    def read(self, size=-1):
+        """
+        Read and return up to size bytes, where size is an int.
+
+        If the argument is omitted, None, or negative, reads and
+        returns all data until EOF.
+
+        If the argument is positive, multiple raw reads may be issued to satisfy
+        the byte count (unless EOF is reached first).
+
+        Returns an empty bytes array on EOF.
+        """
+        if self._current_pos == self._get_size() or size == 0:
+            return b""
+        elif size < 0:
+            obj = self._bucket.get_object(
+                self._key_name, byte_range=(self._current_pos, None)
+            )
+            self._current_pos = self._get_size()
+        else:
+            obj = self._bucket.get_object(
+                self._key_name,
+                byte_range=(self._current_pos, self._current_pos + size - 1),
+            )
+            self._current_pos = self._current_pos + size
+        content = obj.read()
+        return content
+
+    def readline(self, size=-1):
+        # For backwards compatibility, a (slowish) readline().
+        def nreadahead():
+            # Read to the beginning of the next line
+            read_to = min(
+                self._get_size() - 1, self._current_pos + self._buffer_size - 1
+            )
+            buffer = self._bucket.get_object(
+                self._key_name, byte_range=(self._current_pos, read_to)
+            ).read()
+            if not buffer:
+                return 1
+            n = (buffer.find(b"\n") + 1) or len(buffer)
+            if size >= 0:
+                n = min(n, size)
+            return n
+
+        if size is None:
+            size = -1
+        else:
+            try:
+                size_index = size.__index__
+            except AttributeError:
+                raise TypeError(f"{size!r} is not an integer")
+            else:
+                size = size_index()
+        res = bytearray()
+        while size < 0 or len(res) < size:
+            b = self.read(nreadahead())
+            if not b:
+                break
+            res += b
+            if res.endswith(b"\n"):
+                break
+        return bytes(res)
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return False
+
+    def close(self):
+        # already closed by oss
+        pass