maxframe 2.3.0__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

maxframe/lib/filesystem/core.py

@@ -0,0 +1,95 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob as glob_
+import os
+from typing import Dict, List
+from urllib.parse import urlparse
+
+from ..compression import compress
+from .base import FileSystem, path_type
+from .local import LocalFileSystem
+from .oss import OSSFileSystem
+
+_filesystems = {"file": LocalFileSystem, "oss": OSSFileSystem}
+_scheme_to_dependencies = {
+    "hdfs": ["pyarrow"],
+    "az": ["fsspec", "adlfs"],
+    "abfs": ["fsspec", "adlfs"],
+    "s3": ["fsspec", "s3fs"],
+}
+
+
+def register_filesystem(name: str, fs):
+    _filesystems[name] = fs
+
+
+def get_fs(path: path_type, storage_options: Dict = None) -> FileSystem:
+    if storage_options is None:
+        storage_options = dict()
+
+    # detect scheme
+    if os.path.exists(path) or glob_.glob(path):
+        scheme = "file"
+    else:
+        scheme = urlparse(path).scheme
+    if scheme == "" or len(scheme) == 1:  # len == 1 for windows
+        scheme = "file"
+
+    if scheme in _filesystems:
+        file_system_type = _filesystems[scheme]
+        if scheme == "file":
+            # local file systems are singletons.
+            return file_system_type.get_instance()
+        else:
+            storage_options = file_system_type.get_storage_options(
+                storage_options, path
+            )
+            return file_system_type(**storage_options)
+    elif scheme in _scheme_to_dependencies:  # pragma: no cover
+        dependencies = ", ".join(_scheme_to_dependencies[scheme])
+        raise ImportError(f"Need to install {dependencies} to access {scheme}.")
+    else:
+        raise ValueError(
+            f"Unknown file system type: {scheme}, "
+            f'available include: {", ".join(_scheme_to_dependencies.keys())}'
+        )
+
+
+def glob(path: path_type, storage_options: Dict = None) -> List[path_type]:
+    if "*" in path:
+        fs = get_fs(path, storage_options)
+        return fs.glob(path)
+    else:
+        return [path]
+
+
+def file_size(path: path_type, storage_options: Dict = None) -> int:
+    fs = get_fs(path, storage_options)
+    return fs.stat(path)["size"]
+
+
+def open_file(
+    path: path_type,
+    mode: str = "rb",
+    compression: str = None,
+    storage_options: Dict = None,
+):
+    fs = get_fs(path, storage_options)
+    file = fs.open(path, mode=mode)
+
+    if compression is not None:
+        file = compress(file, compression)
+
+    return file

maxframe/lib/filesystem/fsmap.py

@@ -0,0 +1,164 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import MutableMapping
+from urllib.parse import urlparse
+
+from .local import LocalFileSystem
+
+
+class FSMap(MutableMapping):
+    """
+    Wrap a FileSystem instance as a mutable wrapping.
+    The keys of the mapping become files under the given root, and the
+    values (which must be bytes) the contents of those files.
+
+    Parameters
+    ----------
+    root: string
+        prefix for all the files
+    fs: FileSystem instance
+    check: bool (=True)
+        performs a touch at the location, to check for write access.
+    """
+
+    def __init__(self, root, fs, check=False, create=False):
+        self.fs = fs
+        self.root = self._get_path(fs, root)
+        if create:
+            if not self.fs.exists(root):
+                self.fs.mkdir(root)
+        if check:
+            if not self.fs.exists(root):
+                raise ValueError(
+                    f"Path {root} does not exist. Create with the ``create=True`` keyword"
+                )
+            with self.fs.open(fs.pathsep.join([root, "a"]), "w"):
+                pass
+            self.fs.rm(fs.pathsep.join([root, "a"]))
+
+    @staticmethod
+    def _get_path(fs, path):
+        return path if isinstance(fs, LocalFileSystem) else urlparse(path).path
+
+    @staticmethod
+    def _normalize_path(fs, path, lstrip=False, rstrip=False):
+        if fs.pathsep != "/":  # pragma: no cover
+            path = path.replace("/", fs.pathsep)
+        if lstrip:
+            path = path.lstrip(fs.pathsep)
+        if rstrip:
+            path = path.rstrip(fs.pathsep)
+        return path
+
+    @staticmethod
+    def _join_path(fs, paths):
+        if fs.pathsep == "/":
+            return "/".join(paths)
+
+        new_paths = []
+        for i, path in enumerate(paths):
+            path = FSMap._normalize_path(
+                fs, path, lstrip=i > 0, rstrip=i < len(paths) - 1
+            )
+            new_paths.append(path)
+        return fs.pathsep.join(new_paths)
+
+    def clear(self):
+        """Remove all keys below root - empties out mapping"""
+        try:
+            self.fs.rm(self.root, True)
+            self.fs.mkdir(self.root)
+        except:  # noqa: E722  # pragma: no cover
+            pass
+
+    def _key_to_str(self, key):
+        """Generate full path for the key"""
+        if isinstance(key, (tuple, list)):
+            key = str(tuple(key))
+        else:
+            key = str(key)
+        return self._join_path(self.fs, [self.root, key]) if self.root else key
+
+    def _str_to_key(self, s):
+        """Strip path of to leave key name"""
+        key = self._normalize_path(self.fs, s[len(self.root) :], lstrip=True)
+        if self.fs.pathsep != "/":  # pragma: no cover
+            key = key.replace(self.fs.pathsep, "/")
+        return key
+
+    def __getitem__(self, key, default=None):
+        """Retrieve data"""
+        key = self._key_to_str(key)
+        try:
+            result = self.fs.cat(key)
+        except:  # noqa: E722
+            if default is not None:
+                return default
+            raise KeyError(key)
+        return result
+
+    def pop(self, key, default=None):
+        result = self.__getitem__(key, default)
+        try:
+            del self[key]
+        except KeyError:
+            pass
+        return result
+
+    @staticmethod
+    def _parent(fs, path):
+        path = FSMap._get_path(fs, path.rstrip(fs.pathsep))
+        if fs.pathsep in path:
+            return path.rsplit(fs.pathsep, 1)[0]
+        else:  # pragma: no cover
+            return ""
+
+    def __setitem__(self, key, value):
+        """Store value in key"""
+        key = self._key_to_str(key)
+        try:
+            self.fs.mkdir(self._parent(self.fs, key))
+        except FileExistsError:
+            pass
+        with self.fs.open(key, "wb") as f:
+            f.write(value)
+
+    @staticmethod
+    def _find(fs, path):
+        out = set()
+        for path, dirs, files in fs.walk(path):
+            out.update(fs.pathsep.join([path, f]) for f in files)
+        if fs.isfile(path) and path not in out:
+            # walk works on directories, but find should also return [path]
+            # when path happens to be a file
+            out.add(path)
+        return sorted(out)
+
+    def __iter__(self):
+        return (self._str_to_key(x) for x in self._find(self.fs, self.root))
+
+    def __len__(self):
+        return len(self._find(self.fs, self.root))
+
+    def __delitem__(self, key):
+        """Remove key"""
+        try:
+            self.fs.rm(self._key_to_str(key))
+        except:  # noqa: E722
+            raise KeyError
+
+    def __contains__(self, key):
+        """Does key exist in mapping?"""
+        return self.fs.exists(self._key_to_str(key))

maxframe/lib/filesystem/hdfs.py

@@ -0,0 +1,31 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+    from pyarrow.fs import HadoopFileSystem as _ArrowHadoopFileSystem
+
+    from .arrow import HadoopFileSystem
+
+    del _ArrowHadoopFileSystem
+except ImportError:  # pragma: no cover
+    try:
+        # pyarrow < 2.0.0
+        from pyarrow import HadoopFileSystem
+    except ImportError:
+        HadoopFileSystem = None
+
+from .core import register_filesystem
+
+if HadoopFileSystem is not None:  # pragma: no branch
+    register_filesystem("hdfs", HadoopFileSystem)

maxframe/lib/filesystem/local.py

@@ -0,0 +1,112 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+import shutil
+from typing import BinaryIO, Dict, Iterator, List, TextIO, Tuple, Union
+
+from ...utils import implements, stringify_path
+from .base import FileSystem, path_type
+
+
+class LocalFileSystem(FileSystem):
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = LocalFileSystem()
+        return cls._instance
+
+    @implements(FileSystem.cat)
+    def cat(self, path: path_type):
+        with self.open(path, "rb") as f:
+            return f.read()
+
+    @implements(FileSystem.ls)
+    def ls(self, path: path_type) -> List[path_type]:
+        path = stringify_path(path)
+        return sorted(os.path.join(path, x) for x in os.listdir(path))
+
+    @implements(FileSystem.delete)
+    def delete(self, path: path_type, recursive: bool = False):
+        if os.path.isfile(path):
+            os.remove(path)
+        elif not recursive:
+            os.rmdir(path)
+        else:
+            shutil.rmtree(path)
+
+    @implements(FileSystem.rename)
+    def rename(self, path: path_type, new_path: path_type):
+        os.rename(path, new_path)
+
+    @implements(FileSystem.stat)
+    def stat(self, path: path_type) -> Dict:
+        os_stat = os.stat(path)
+        stat = dict(name=path, size=os_stat.st_size, modified_time=os_stat.st_mtime)
+        if os.path.isfile(path):
+            stat["type"] = "file"
+        elif os.path.isdir(path):
+            stat["type"] = "directory"
+        else:  # pragma: no cover
+            stat["type"] = "other"
+        return stat
+
+    @implements(FileSystem.mkdir)
+    def mkdir(self, path: path_type, create_parents: bool = True):
+        path = stringify_path(path)
+        if create_parents:
+            os.makedirs(path)
+        else:
+            os.mkdir(path)
+
+    @implements(FileSystem.isdir)
+    def isdir(self, path: path_type) -> bool:
+        path = stringify_path(path)
+        return os.path.isdir(path)
+
+    @implements(FileSystem.isfile)
+    def isfile(self, path: path_type) -> bool:
+        path = stringify_path(path)
+        return os.path.isfile(path)
+
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self) -> bool:
+        return True
+
+    @implements(FileSystem.exists)
+    def exists(self, path: path_type):
+        path = stringify_path(path)
+        return os.path.exists(path)
+
+    @implements(FileSystem.open)
+    def open(self, path: path_type, mode: str = "rb") -> Union[BinaryIO, TextIO]:
+        path = stringify_path(path)
+        return open(path, mode=mode)
+
+    @implements(FileSystem.walk)
+    def walk(self, path: path_type) -> Iterator[Tuple[str, List[str], List[str]]]:
+        path = stringify_path(path)
+        return os.walk(path)
+
+    @implements(FileSystem.glob)
+    def glob(self, path: path_type, recursive: bool = False) -> List[path_type]:
+        path = stringify_path(path)
+        return glob.glob(path, recursive=recursive)
+
+    @property
+    def pathsep(self) -> str:
+        return os.path.sep

maxframe/lib/filesystem/oss.py

@@ -0,0 +1,226 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+import re
+from typing import Dict, Iterator, List, Tuple, Union
+from urllib.parse import urlencode
+
+from ...utils import implements, lazy_import
+from ._oss_lib import common as oc
+from ._oss_lib.glob import glob
+from ._oss_lib.handle import OSSIOBase
+from .base import FileSystem, path_type
+
+oss2 = lazy_import("oss2", placeholder=True)
+_ip_regex = re.compile(r"^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})")
+
+_oss_time_out = 10
+
+
+class HostEnforceType(enum.Enum):
+    no_enforce = 0
+    force_internal = 1
+    force_external = 2
+
+
+class OSSFileSystem(FileSystem):
+    def __init__(
+        self,
+        access_key_id: str = None,
+        access_key_secret: str = None,
+        security_token: str = None,
+        host_enforce_type: Union[HostEnforceType, str] = HostEnforceType.no_enforce,
+        **kw,
+    ):
+        self._access_key_id = access_key_id or kw.get("user")
+        self._access_key_secret = access_key_secret or kw.get("password")
+        self._security_token = security_token
+        self._host_enforce_type = (
+            host_enforce_type
+            if isinstance(host_enforce_type, HostEnforceType)
+            else getattr(HostEnforceType, host_enforce_type)
+        )
+
+    def _rewrite_path(self, path: str) -> str:
+        return build_oss_path(
+            path,
+            access_key_id=self._access_key_id,
+            access_key_secret=self._access_key_secret,
+            security_token=self._security_token,
+            host_enforce_type=self._host_enforce_type,
+        )
+
+    @implements(FileSystem.cat)
+    def cat(self, path: path_type):
+        raise NotImplementedError
+
+    @implements(FileSystem.ls)
+    def ls(self, path: path_type) -> List[path_type]:
+        file_list = []
+        file_entry = oc.OSSFileEntry(path)
+        if not file_entry.is_dir():
+            raise OSError("ls for file is not supported")
+        else:
+            parsed_path = oc.parse_osspath(path)
+            oss_bucket = oc.get_oss_bucket(parsed_path)
+            for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=parsed_path.key):
+                if obj.key.endswith("/"):
+                    continue
+                obj_path = rf"{parsed_path.bucket}/{obj.key}"
+                file_list.append(
+                    build_oss_path(
+                        obj_path,
+                        parsed_path.endpoint,
+                        parsed_path.access_key_id,
+                        parsed_path.access_key_secret,
+                        parsed_path.security_token,
+                    )
+                )
+        return file_list
+
+    @implements(FileSystem.delete)
+    def delete(self, path: path_type, recursive: bool = False):
+        return oc.oss_delete(self._rewrite_path(path))
+
+    @implements(FileSystem.rename)
+    def rename(self, path: path_type, new_path: path_type):
+        # in OSS, you need to move file by copy and delete
+        path = self._rewrite_path(path)
+        oc.oss_copy_file(path, self._rewrite_path(new_path))
+        oc.oss_delete(path)
+
+    @implements(FileSystem.stat)
+    def stat(self, path: path_type) -> Dict:
+        ofe = oc.OSSFileEntry(self._rewrite_path(path))
+        return ofe.stat()
+
+    @implements(FileSystem.mkdir)
+    def mkdir(self, path: path_type, create_parents: bool = True):
+        raise NotImplementedError
+
+    @implements(FileSystem.isdir)
+    def isdir(self, path: path_type) -> bool:
+        file_entry = oc.OSSFileEntry(self._rewrite_path(path))
+        return file_entry.is_dir()
+
+    @implements(FileSystem.isfile)
+    def isfile(self, path: path_type) -> bool:
+        file_entry = oc.OSSFileEntry(self._rewrite_path(path))
+        return file_entry.is_file()
+
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self) -> bool:
+        raise NotImplementedError
+
+    @implements(FileSystem.exists)
+    def exists(self, path: path_type):
+        return oc.oss_exists(self._rewrite_path(path))
+
+    @implements(FileSystem.open)
+    def open(self, path: path_type, mode: str = "rb") -> OSSIOBase:
+        file_handle = OSSIOBase(self._rewrite_path(path), mode)
+        return file_handle
+
+    @implements(FileSystem.walk)
+    def walk(self, path: path_type) -> Iterator[Tuple[str, List[str], List[str]]]:
+        raise NotImplementedError
+
+    @implements(FileSystem.glob)
+    def glob(self, path: path_type, recursive: bool = False) -> List[path_type]:
+        return glob(self._rewrite_path(path), recursive=recursive)
+
+
+def _rewrite_internal_endpoint(
+    endpoint: str, host_enforce_type: HostEnforceType = HostEnforceType.no_enforce
+) -> str:
+    if (
+        not endpoint
+        or host_enforce_type == HostEnforceType.no_enforce
+        or _ip_regex.match(endpoint)
+    ):
+        return endpoint
+
+    ep_first, ep_rest = endpoint.split(".", 1)
+    host_with_internal = ep_first.endswith("-internal")
+    if host_enforce_type == HostEnforceType.force_external and host_with_internal:
+        return ep_first.replace("-internal", "") + "." + ep_rest
+    elif host_enforce_type == HostEnforceType.force_internal and not host_with_internal:
+        return ep_first + "-internal." + ep_rest
+    else:
+        return endpoint
+
+
+def build_oss_path(
+    path: path_type,
+    endpoint: str = None,
+    access_key_id: str = None,
+    access_key_secret: str = None,
+    security_token: str = None,
+    host_enforce_type: HostEnforceType = HostEnforceType.no_enforce,
+):
+    """
+    Returns a path with oss info.
+    Used to register the access_key_id, access_key_secret and
+    endpoint of OSS. The access_key_id and endpoint are put
+    into the url with url-safe-base64 encoding.
+
+    Parameters
+    ----------
+    path : path_type
+        The original OSS url.
+
+    endpoint : str
+        The endpoint of OSS.
+
+    access_key_id : str
+        The access key id of OSS.
+
+    access_key_secret : str
+        The access key secret of OSS.
+
+    security_token : str
+        The security token of OSS.
+
+    Returns
+    -------
+    path_type
+        Path include the encoded access key id, end point and
+        access key secret of oss.
+    """
+    if isinstance(path, (list, tuple)):
+        path = path[0]
+    parse_result = oc.parse_osspath(path, check_errors=False)
+    access_key_id = parse_result.access_key_id or access_key_id
+    access_key_secret = parse_result.access_key_secret or access_key_secret
+    security_token = parse_result.security_token or security_token
+
+    scheme = parse_result.scheme or "oss"
+    endpoint = _rewrite_internal_endpoint(
+        parse_result.endpoint or endpoint, host_enforce_type
+    )
+
+    if access_key_id and access_key_secret:
+        creds = f"{access_key_id}:{access_key_secret}@"
+    else:
+        creds = ""
+
+    new_path = f"{scheme}://{creds}{endpoint}/{parse_result.bucket}"
+    if parse_result.key:
+        new_path += f"/{parse_result.key}"
+    if security_token:
+        new_path += f"?{urlencode(dict(security_token=security_token))}"
+    # reparse to check errors
+    oc.parse_osspath(new_path)
+    return new_path

maxframe/lib/filesystem/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.