maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/lib/filesystem/fsmap.py

@@ -0,0 +1,164 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import MutableMapping
+from urllib.parse import urlparse
+
+from .local import LocalFileSystem
+
+
+class FSMap(MutableMapping):
+    """
+    Wrap a FileSystem instance as a mutable wrapping.
+    The keys of the mapping become files under the given root, and the
+    values (which must be bytes) the contents of those files.
+
+    Parameters
+    ----------
+    root: string
+        prefix for all the files
+    fs: FileSystem instance
+    check: bool (=True)
+        performs a touch at the location, to check for write access.
+    """
+
+    def __init__(self, root, fs, check=False, create=False):
+        self.fs = fs
+        self.root = self._get_path(fs, root)
+        if create:
+            if not self.fs.exists(root):
+                self.fs.mkdir(root)
+        if check:
+            if not self.fs.exists(root):
+                raise ValueError(
+                    f"Path {root} does not exist. Create with the ``create=True`` keyword"
+                )
+            with self.fs.open(fs.pathsep.join([root, "a"]), "w"):
+                pass
+            self.fs.rm(fs.pathsep.join([root, "a"]))
+
+    @staticmethod
+    def _get_path(fs, path):
+        return path if isinstance(fs, LocalFileSystem) else urlparse(path).path
+
+    @staticmethod
+    def _normalize_path(fs, path, lstrip=False, rstrip=False):
+        if fs.pathsep != "/":  # pragma: no cover
+            path = path.replace("/", fs.pathsep)
+        if lstrip:
+            path = path.lstrip(fs.pathsep)
+        if rstrip:
+            path = path.rstrip(fs.pathsep)
+        return path
+
+    @staticmethod
+    def _join_path(fs, paths):
+        if fs.pathsep == "/":
+            return "/".join(paths)
+
+        new_paths = []
+        for i, path in enumerate(paths):
+            path = FSMap._normalize_path(
+                fs, path, lstrip=i > 0, rstrip=i < len(paths) - 1
+            )
+            new_paths.append(path)
+        return fs.pathsep.join(new_paths)
+
+    def clear(self):
+        """Remove all keys below root - empties out mapping"""
+        try:
+            self.fs.rm(self.root, True)
+            self.fs.mkdir(self.root)
+        except:  # noqa: E722  # pragma: no cover
+            pass
+
+    def _key_to_str(self, key):
+        """Generate full path for the key"""
+        if isinstance(key, (tuple, list)):
+            key = str(tuple(key))
+        else:
+            key = str(key)
+        return self._join_path(self.fs, [self.root, key]) if self.root else key
+
+    def _str_to_key(self, s):
+        """Strip path of to leave key name"""
+        key = self._normalize_path(self.fs, s[len(self.root) :], lstrip=True)
+        if self.fs.pathsep != "/":  # pragma: no cover
+            key = key.replace(self.fs.pathsep, "/")
+        return key
+
+    def __getitem__(self, key, default=None):
+        """Retrieve data"""
+        key = self._key_to_str(key)
+        try:
+            result = self.fs.cat(key)
+        except:  # noqa: E722
+            if default is not None:
+                return default
+            raise KeyError(key)
+        return result
+
+    def pop(self, key, default=None):
+        result = self.__getitem__(key, default)
+        try:
+            del self[key]
+        except KeyError:
+            pass
+        return result
+
+    @staticmethod
+    def _parent(fs, path):
+        path = FSMap._get_path(fs, path.rstrip(fs.pathsep))
+        if fs.pathsep in path:
+            return path.rsplit(fs.pathsep, 1)[0]
+        else:  # pragma: no cover
+            return ""
+
+    def __setitem__(self, key, value):
+        """Store value in key"""
+        key = self._key_to_str(key)
+        try:
+            self.fs.mkdir(self._parent(self.fs, key))
+        except FileExistsError:
+            pass
+        with self.fs.open(key, "wb") as f:
+            f.write(value)
+
+    @staticmethod
+    def _find(fs, path):
+        out = set()
+        for path, dirs, files in fs.walk(path):
+            out.update(fs.pathsep.join([path, f]) for f in files)
+        if fs.isfile(path) and path not in out:
+            # walk works on directories, but find should also return [path]
+            # when path happens to be a file
+            out.add(path)
+        return sorted(out)
+
+    def __iter__(self):
+        return (self._str_to_key(x) for x in self._find(self.fs, self.root))
+
+    def __len__(self):
+        return len(self._find(self.fs, self.root))
+
+    def __delitem__(self, key):
+        """Remove key"""
+        try:
+            self.fs.rm(self._key_to_str(key))
+        except:  # noqa: E722
+            raise KeyError
+
+    def __contains__(self, key):
+        """Does key exist in mapping?"""
+        return self.fs.exists(self._key_to_str(key))

maxframe/lib/filesystem/hdfs.py

@@ -0,0 +1,31 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+    from pyarrow.fs import HadoopFileSystem as _ArrowHadoopFileSystem
+
+    from .arrow import HadoopFileSystem
+
+    del _ArrowHadoopFileSystem
+except ImportError:  # pragma: no cover
+    try:
+        # pyarrow < 2.0.0
+        from pyarrow import HadoopFileSystem
+    except ImportError:
+        HadoopFileSystem = None
+
+from .core import register_filesystem
+
+if HadoopFileSystem is not None:  # pragma: no branch
+    register_filesystem("hdfs", HadoopFileSystem)

maxframe/lib/filesystem/local.py

@@ -0,0 +1,112 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+import shutil
+from typing import BinaryIO, Dict, Iterator, List, TextIO, Tuple, Union
+
+from ...utils import implements, stringify_path
+from .base import FileSystem, path_type
+
+
+class LocalFileSystem(FileSystem):
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = LocalFileSystem()
+        return cls._instance
+
+    @implements(FileSystem.cat)
+    def cat(self, path: path_type):
+        with self.open(path, "rb") as f:
+            return f.read()
+
+    @implements(FileSystem.ls)
+    def ls(self, path: path_type) -> List[path_type]:
+        path = stringify_path(path)
+        return sorted(os.path.join(path, x) for x in os.listdir(path))
+
+    @implements(FileSystem.delete)
+    def delete(self, path: path_type, recursive: bool = False):
+        if os.path.isfile(path):
+            os.remove(path)
+        elif not recursive:
+            os.rmdir(path)
+        else:
+            shutil.rmtree(path)
+
+    @implements(FileSystem.rename)
+    def rename(self, path: path_type, new_path: path_type):
+        os.rename(path, new_path)
+
+    @implements(FileSystem.stat)
+    def stat(self, path: path_type) -> Dict:
+        os_stat = os.stat(path)
+        stat = dict(name=path, size=os_stat.st_size, modified_time=os_stat.st_mtime)
+        if os.path.isfile(path):
+            stat["type"] = "file"
+        elif os.path.isdir(path):
+            stat["type"] = "directory"
+        else:  # pragma: no cover
+            stat["type"] = "other"
+        return stat
+
+    @implements(FileSystem.mkdir)
+    def mkdir(self, path: path_type, create_parents: bool = True):
+        path = stringify_path(path)
+        if create_parents:
+            os.makedirs(path)
+        else:
+            os.mkdir(path)
+
+    @implements(FileSystem.isdir)
+    def isdir(self, path: path_type) -> bool:
+        path = stringify_path(path)
+        return os.path.isdir(path)
+
+    @implements(FileSystem.isfile)
+    def isfile(self, path: path_type) -> bool:
+        path = stringify_path(path)
+        return os.path.isfile(path)
+
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self) -> bool:
+        return True
+
+    @implements(FileSystem.exists)
+    def exists(self, path: path_type):
+        path = stringify_path(path)
+        return os.path.exists(path)
+
+    @implements(FileSystem.open)
+    def open(self, path: path_type, mode: str = "rb") -> Union[BinaryIO, TextIO]:
+        path = stringify_path(path)
+        return open(path, mode=mode)
+
+    @implements(FileSystem.walk)
+    def walk(self, path: path_type) -> Iterator[Tuple[str, List[str], List[str]]]:
+        path = stringify_path(path)
+        return os.walk(path)
+
+    @implements(FileSystem.glob)
+    def glob(self, path: path_type, recursive: bool = False) -> List[path_type]:
+        path = stringify_path(path)
+        return glob.glob(path, recursive=recursive)
+
+    @property
+    def pathsep(self) -> str:
+        return os.path.sep

maxframe/lib/filesystem/oss.py

@@ -0,0 +1,157 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Iterator, List, Tuple
+from urllib import parse
+
+from ...utils import implements, lazy_import
+from ._oss_lib import common as oc
+from ._oss_lib.glob import glob
+from ._oss_lib.handle import OSSIOBase
+from .base import FileSystem, path_type
+
+oss2 = lazy_import("oss2", placeholder=True)
+
+_oss_time_out = 10
+
+
+class OSSFileSystem(FileSystem):
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = OSSFileSystem()
+        return cls._instance
+
+    @implements(FileSystem.cat)
+    def cat(self, path: path_type):
+        raise NotImplementedError
+
+    @implements(FileSystem.ls)
+    def ls(self, path: path_type) -> List[path_type]:
+        file_list = []
+        file_entry = oc.OSSFileEntry(path)
+        if not file_entry.is_dir():
+            raise OSError("ls for file is not supported")
+        else:
+            bucket, key, access_key_id, access_key_secret, end_point = oc.parse_osspath(
+                path
+            )
+            oss_bucket = oss2.Bucket(
+                auth=oss2.Auth(
+                    access_key_id=access_key_id, access_key_secret=access_key_secret
+                ),
+                endpoint=end_point,
+                bucket_name=bucket,
+                connect_timeout=_oss_time_out,
+            )
+            for obj in oss2.ObjectIteratorV2(oss_bucket, prefix=key):
+                if obj.key.endswith("/"):
+                    continue
+                obj_path = rf"oss://{bucket}/{obj.key}"
+                file_list.append(
+                    build_oss_path(
+                        obj_path, access_key_id, access_key_secret, end_point
+                    )
+                )
+        return file_list
+
+    @implements(FileSystem.delete)
+    def delete(self, path: path_type, recursive: bool = False):
+        raise NotImplementedError
+
+    @implements(FileSystem.rename)
+    def rename(self, path: path_type, new_path: path_type):
+        raise NotImplementedError
+
+    @implements(FileSystem.stat)
+    def stat(self, path: path_type) -> Dict:
+        ofe = oc.OSSFileEntry(path)
+        return ofe.stat()
+
+    @implements(FileSystem.mkdir)
+    def mkdir(self, path: path_type, create_parents: bool = True):
+        raise NotImplementedError
+
+    @implements(FileSystem.isdir)
+    def isdir(self, path: path_type) -> bool:
+        file_entry = oc.OSSFileEntry(path)
+        return file_entry.is_dir()
+
+    @implements(FileSystem.isfile)
+    def isfile(self, path: path_type) -> bool:
+        file_entry = oc.OSSFileEntry(path)
+        return file_entry.is_file()
+
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self) -> bool:
+        raise NotImplementedError
+
+    @implements(FileSystem.exists)
+    def exists(self, path: path_type):
+        return oc.oss_exists(path)
+
+    @implements(FileSystem.open)
+    def open(self, path: path_type, mode: str = "rb") -> OSSIOBase:
+        file_handle = OSSIOBase(path, mode)
+        return file_handle
+
+    @implements(FileSystem.walk)
+    def walk(self, path: path_type) -> Iterator[Tuple[str, List[str], List[str]]]:
+        raise NotImplementedError
+
+    @implements(FileSystem.glob)
+    def glob(self, path: path_type, recursive: bool = False) -> List[path_type]:
+        return glob(path, recursive=recursive)
+
+
+def build_oss_path(path: path_type, access_key_id, access_key_secret, end_point):
+    """
+    Returns a path with oss info.
+    Used to register the access_key_id, access_key_secret and
+    endpoint of OSS. The access_key_id and endpoint are put
+    into the url with url-safe-base64 encoding.
+
+    Parameters
+    ----------
+    path : path_type
+        The original oss url.
+
+    access_key_id : str
+        The access key id of oss.
+
+    access_key_secret : str
+        The access key secret of oss.
+
+    end_point : str
+        The endpoint of oss.
+
+    Returns
+    -------
+    path_type
+        Path include the encoded access key id, end point and
+        access key secret of oss.
+    """
+    if isinstance(path, (list, tuple)):
+        path = path[0]
+    param_dict = {"access_key_id": access_key_id, "end_point": end_point}
+    id_endpoint = oc.dict_to_url(param_dict)
+    password = access_key_secret
+    parse_result = parse.urlparse(path)
+    new_path = (
+        f"{parse_result.scheme}://{id_endpoint}:{password}"
+        f"@{parse_result.netloc}{parse_result.path}"
+    )
+    return new_path

maxframe/lib/filesystem/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/lib/filesystem/tests/test_filesystem.py

@@ -0,0 +1,223 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob as _glob
+import os
+import tempfile
+
+import numpy as np
+import pytest
+
+try:
+    import pyarrow as pa
+except ImportError:  # pragma: no cover
+    pa = None
+
+from ....tests.utils import require_hadoop
+from ....utils import lazy_import
+from .. import FileSystem, FSMap, LocalFileSystem, glob
+
+if pa is not None:
+    from ..arrow import ArrowBasedLocalFileSystem, HadoopFileSystem
+else:  # pragma: no cover
+    ArrowBasedLocalFileSystem = None
+
+fsspec_installed = lazy_import("fsspec") is not None
+
+
+def test_path_parser():
+    path = "hdfs://user:password@localhost:8080/test"
+    parsed_result = FileSystem.parse_from_path(path)
+    assert parsed_result["host"] == "localhost"
+    assert parsed_result["port"] == 8080
+    assert parsed_result["user"] == "user"
+    assert parsed_result["password"] == "password"
+
+
+def test_local_filesystem():
+    local_fs1 = LocalFileSystem.get_instance()
+    local_fs2 = LocalFileSystem.get_instance()
+    assert local_fs1 is local_fs2
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        file_path = os.path.join(tempdir, "test")
+
+        with open(file_path, "wb") as f:
+            f.write(b"text for test")
+        assert local_fs1.stat(tempdir)["type"] == "directory"
+        assert local_fs1.stat(file_path)["type"] == "file"
+        assert len(glob(tempdir + "*")) == 1
+
+
+@pytest.mark.parametrize(
+    "fs_type",
+    [LocalFileSystem, ArrowBasedLocalFileSystem]
+    if pa is not None
+    else [LocalFileSystem],
+)
+def test_filesystems(fs_type):
+    fs = fs_type.get_instance()
+
+    with tempfile.TemporaryDirectory() as root:
+        test1_dir = os.path.join(root, "test1")
+        fs.mkdir(test1_dir, create_parents=False)
+        test2_dir = os.path.join(root, "test2")
+        sub_test2_dir = os.path.join(test2_dir, "sub_test2")
+        fs.mkdir(sub_test2_dir)
+
+        sub_test2_dir_stat = fs.stat(sub_test2_dir)
+        assert sub_test2_dir_stat["type"] == "directory"
+        assert sub_test2_dir_stat["name"] == sub_test2_dir
+        assert fs.isdir(sub_test2_dir)
+
+        test1_file = os.path.join(test1_dir, "test1")
+        with fs.open(test1_file, "wb") as f:
+            f.write(b"abc test")
+        with fs.open(test1_file, "ab") as f:
+            f.write(b"\nappend test")
+        with fs.open(test1_file, "rb") as f:
+            content = f.read()
+            with open(test1_file, "rb") as f2:
+                expected = f2.read()
+                assert content == expected
+
+        assert fs.cat(test1_file) == expected
+
+        assert fs.isfile(test1_file)
+        test1_file_stat = fs.stat(test1_file)
+        assert test1_file_stat["type"] == "file"
+        assert test1_file_stat["name"] == test1_file
+        assert test1_file_stat["size"] == os.stat(test1_file).st_size
+        np.testing.assert_almost_equal(
+            test1_file_stat["modified_time"], os.stat(test1_file).st_mtime, decimal=6
+        )
+
+        walked = [
+            (os.path.normpath(root), dirs, files) for root, dirs, files in fs.walk(root)
+        ]
+        expected = os.walk(root)
+        assert sorted(walked) == sorted(expected)
+
+        test2_file = os.path.join(sub_test2_dir, "test2")
+        with fs.open(test2_file, "wb") as f:
+            f.write(b"def test")
+
+        for recursive in [False, True]:
+            globs = [
+                os.path.normpath(p)
+                for p in fs.glob(os.path.join(root, "*"), recursive=recursive)
+            ]
+            expected = [
+                os.path.normpath(p)
+                for p in _glob.glob(os.path.join(root, "*"), recursive=recursive)
+            ]
+            assert sorted(globs) == sorted(expected)
+
+        for path in [os.path.join(root, "*", "*"), test1_dir]:
+            globs = [os.path.normpath(p) for p in fs.glob(path)]
+            expected = [os.path.normpath(p) for p in _glob.glob(path)]
+            assert sorted(globs) == sorted(expected)
+
+        test1_new_file = os.path.join(test1_dir, "test1_new")
+        fs.rename(test1_file, test1_new_file)
+        test1_new_file2 = os.path.join(test1_dir, "test1_new2")
+        fs.mv(test1_new_file, test1_new_file2)
+        assert fs.exists(test1_new_file2)
+        assert not fs.exists(test1_file)
+
+        assert fs.disk_usage(test1_dir) > 0
+
+        fs.delete(test2_file)
+        assert not fs.exists(test2_file)
+
+        assert fs._isfilestore()
+
+        with pytest.raises(OSError):
+            fs.delete(test1_dir)
+        fs.delete(test1_dir, recursive=True)
+        assert not fs.exists(test1_dir)
+
+
+@require_hadoop
+def test_hadoop_filesystem():
+    fs = HadoopFileSystem(host="localhost", port=8020)
+
+    test_dir = "/tmp/test/test_hadoop_fs"
+    fs.mkdir(test_dir)
+    test_file = f"{test_dir}/my_file.txt"
+    test_file_content = b"text for text"
+    with fs.open(test_file, "wb") as f:
+        f.write(test_file_content)
+    with fs.open(test_file, "rb") as f:
+        assert test_file_content == f.read()
+    # test file with hdfs:// prefix
+    assert fs.exists(f"hdfs://{test_dir}")
+
+
+def test_fsmap():
+    fs = LocalFileSystem.get_instance()
+    with tempfile.TemporaryDirectory() as root:
+        fs_map = FSMap(root, fs, check=True)
+
+        path = "/to/path/test_file"
+        test_content = b"text for test"
+        fs_map[path] = test_content
+        assert fs_map[path] == test_content
+        assert len(fs_map) == 1
+        assert path in fs_map
+
+        path2 = "/to/path2/test_file2"
+        fs_map[path2] = test_content
+        assert len(fs_map) == 2
+
+        del fs_map[path]
+        assert list(fs_map) == ["to/path2/test_file2"]
+
+        path3 = "/to2/path3/test_file3"
+        fs_map[path3] = test_content
+        assert fs_map.pop(path3) == test_content
+        assert fs_map.pop(path3, "fake_content") == "fake_content"
+        with pytest.raises(KeyError):
+            fs_map.pop("not_exist")
+
+        fs_map.clear()
+        assert len(fs_map) == 0
+
+        # test root not exist
+        with pytest.raises(ValueError):
+            _ = FSMap(root + "/path2", fs, check=True)
+
+        # create root
+        fs_map = FSMap(root + "/path2", fs, create=True)
+        assert len(fs_map) == 0
+
+
+@pytest.mark.skipif(not fsspec_installed, reason="fsspec not installed")
+def test_get_fs():
+    from .. import get_fs, register_filesystem
+    from ..fsspec_adapter import FsSpecAdapter
+
+    class InMemoryFileSystemAdapter(FsSpecAdapter):
+        def __init__(self, **kwargs):
+            super().__init__("memory", **kwargs)
+
+    register_filesystem("memory", InMemoryFileSystemAdapter)
+
+    assert isinstance(get_fs("file://"), LocalFileSystem)
+    assert isinstance(get_fs("memory://"), InMemoryFileSystemAdapter)
+
+    try:
+        get_fs("unknown://")
+    except ValueError as e:
+        assert "Unknown file system type" in e.__str__()