rclone-api 1.5.8__py3-none-any.whl

rclone_api/file_part.py

@@ -0,0 +1,198 @@
+import atexit
+import os
+import time
+import warnings
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+_TMP_DIR_ACCESS_LOCK = Lock()
+
+
+def _clean_old_files(out: Path) -> None:
+    # clean up files older than 1 day
+    from rclone_api.util import locked_print
+
+    now = time.time()
+    # Erase all stale files and then purge empty directories.
+    for root, dirs, files in os.walk(out):
+        for name in files:
+            f = Path(root) / name
+            filemod = f.stat().st_mtime
+            diff_secs = now - filemod
+            diff_days = diff_secs / (60 * 60 * 24)
+            if diff_days > 1:
+                locked_print(f"Removing old file: {f}")
+                f.unlink()
+
+    for root, dirs, _ in os.walk(out):
+        for dir in dirs:
+            d = Path(root) / dir
+            if not list(d.iterdir()):
+                locked_print(f"Removing empty directory: {d}")
+                d.rmdir()
+
+
+def get_chunk_tmpdir() -> Path:
+    with _TMP_DIR_ACCESS_LOCK:
+        dat = get_chunk_tmpdir.__dict__
+        if "out" in dat:
+            return dat["out"]  # Folder already validated.
+        out = Path("chunk_store")
+        if out.exists():
+            # first access, clean up directory
+            _clean_old_files(out)
+        out.mkdir(exist_ok=True, parents=True)
+        dat["out"] = out
+        return out
+
+
+_CLEANUP_LIST: list[Path] = []
+
+
+def _add_for_cleanup(path: Path) -> None:
+    _CLEANUP_LIST.append(path)
+
+
+def _on_exit_cleanup() -> None:
+    paths = list(_CLEANUP_LIST)
+    for path in paths:
+        try:
+            if path.exists():
+                path.unlink()
+        except Exception as e:
+            warnings.warn(f"Cannot cleanup {path}: {e}")
+
+
+atexit.register(_on_exit_cleanup)
+
+
+_FILEPARTS: list["FilePart"] = []
+
+_FILEPARTS_LOCK = Lock()
+
+
+def _add_filepart(part: "FilePart") -> None:
+    with _FILEPARTS_LOCK:
+        if part not in _FILEPARTS:
+            _FILEPARTS.append(part)
+
+
+def _remove_filepart(part: "FilePart") -> None:
+    with _FILEPARTS_LOCK:
+        if part in _FILEPARTS:
+            _FILEPARTS.remove(part)
+
+
+def run_debug_parts():
+    while True:
+        print("\nAlive file parts:")
+        for part in list(_FILEPARTS):
+            print(part)
+            # print(part.stacktrace)
+        print("\n\n")
+        time.sleep(60)
+
+
+# dbg_thread = threading.Thread(target=run_debug_parts)
+# dbg_thread.start()
+
+
+class FilePart:
+    def __init__(self, payload: Path | bytes | Exception, extra: Any) -> None:
+        import traceback
+
+        from rclone_api.util import random_str
+
+        stacktrace = traceback.format_stack()
+        stacktrace_str = "".join(stacktrace)
+        self.stacktrace = stacktrace_str
+        # _FILEPARTS.append(self)
+        _add_filepart(self)
+
+        self.extra = extra
+        self._lock = Lock()
+        self.payload: Path | Exception
+        if isinstance(payload, Exception):
+            self.payload = payload
+            return
+        if isinstance(payload, bytes):
+            print(f"Creating file part with payload: {len(payload)}")
+            self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+            with _TMP_DIR_ACCESS_LOCK:
+                if not self.payload.parent.exists():
+                    self.payload.parent.mkdir(parents=True, exist_ok=True)
+                self.payload.write_bytes(payload)
+            _add_for_cleanup(self.payload)
+        if isinstance(payload, Path):
+            print("Adopting payload: ", payload)
+            self.payload = payload
+            _add_for_cleanup(self.payload)
+
+    def get_file(self) -> Path | Exception:
+        return self.payload
+
+    @property
+    def size(self) -> int:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                return self.payload.stat().st_size
+            return -1
+
+    def n_bytes(self) -> int:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                return self.payload.stat().st_size
+            return -1
+
+    def load(self) -> bytes:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                with open(self.payload, "rb") as f:
+                    return f.read()
+            raise ValueError("Cannot load from error")
+
+    def __post_init__(self):
+        if isinstance(self.payload, Path):
+            assert self.payload.exists(), f"File part {self.payload} does not exist"
+            assert self.payload.is_file(), f"File part {self.payload} is not a file"
+            assert self.payload.stat().st_size > 0, f"File part {self.payload} is empty"
+        elif isinstance(self.payload, Exception):
+            warnings.warn(f"File part error: {self.payload}")
+        print(f"File part created with payload: {self.payload}")
+
+    def is_error(self) -> bool:
+        return isinstance(self.payload, Exception)
+
+    def dispose(self) -> None:
+        # _FILEPARTS.remove(self)
+        _remove_filepart(self)
+        print("Disposing file part")
+        with self._lock:
+            if isinstance(self.payload, Exception):
+                warnings.warn(
+                    f"Cannot close file part because the payload represents an error: {self.payload}"
+                )
+                print("Cannot close file part because the payload represents an error")
+                return
+            if self.payload.exists():
+                print(f"File part {self.payload} exists")
+                try:
+                    print(f"Unlinking file part {self.payload}")
+                    self.payload.unlink()
+                    print(f"File part {self.payload} deleted")
+                except Exception as e:
+                    warnings.warn(f"Cannot close file part because of error: {e}")
+            else:
+                warnings.warn(
+                    f"Cannot close file part because it does not exist: {self.payload}"
+                )
+
+    def __del__(self):
+        self.dispose()
+
+    def __repr__(self):
+        from rclone_api.types import SizeSuffix
+
+        payload_str = "err" if self.is_error() else f"{SizeSuffix(self.n_bytes())}"
+        return f"FilePart(payload={payload_str}, extra={self.extra})"

rclone_api/file_stream.py

@@ -0,0 +1,52 @@
+"""
+Unit test file.
+"""
+
+from typing import Generator
+
+from rclone_api.file import FileItem
+from rclone_api.process import Process
+
+
+class FilesStream:
+
+    def __init__(self, path: str, process: Process) -> None:
+        self.path = path
+        self.process = process
+
+    def __enter__(self) -> "FilesStream":
+        self.process.__enter__()
+        return self
+
+    def __exit__(self, *exc_info):
+        self.process.__exit__(*exc_info)
+
+    def files(self) -> Generator[FileItem, None, None]:
+        line: bytes
+        for line in self.process.stdout:
+            linestr: str = line.decode("utf-8").strip()
+            if linestr.startswith("["):
+                continue
+            if linestr.endswith(","):
+                linestr = linestr[:-1]
+            if linestr.endswith("]"):
+                continue
+            fileitem: FileItem | None = FileItem.from_json_str(self.path, linestr)
+            if fileitem is None:
+                continue
+            yield fileitem
+
+    def files_paged(
+        self, page_size: int = 1000
+    ) -> Generator[list[FileItem], None, None]:
+        page: list[FileItem] = []
+        for fileitem in self.files():
+            page.append(fileitem)
+            if len(page) >= page_size:
+                yield page
+                page = []
+        if len(page) > 0:
+            yield page
+
+    def __iter__(self) -> Generator[FileItem, None, None]:
+        return self.files()

rclone_api/filelist.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+
+from rclone_api.dir import Dir
+from rclone_api.dir_listing import DirListing
+from rclone_api.file import File
+from rclone_api.rpath import RPath
+
+
+@dataclass
+class FileList:
+    """Remote file dataclass."""
+
+    dirs: list[Dir]
+    files: list[File]
+
+    def _to_dir_list(self) -> list[RPath]:
+        pathlist: list[RPath] = []
+        for d in self.dirs:
+            pathlist.append(d.path)
+        for f in self.files:
+            pathlist.append(f.path)
+        return pathlist
+
+    def __str__(self) -> str:
+        pathlist: list[RPath] = self._to_dir_list()
+        return str(DirListing(pathlist))
+
+    def __repr__(self) -> str:
+        pathlist: list[RPath] = self._to_dir_list()
+        return repr(DirListing(pathlist))

rclone_api/group_files.py

@@ -0,0 +1,256 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class PrefixResult:
+    prefix: str
+    files: list[str]
+
+
+@dataclass
+class FilePathParts:
+    """File path dataclass."""
+
+    remote: str
+    parents: list[str]
+    name: str
+
+    def to_string(self, include_remote: bool, include_bucket: bool) -> str:
+        """Convert to string, may throw for not include_bucket=False."""
+        parents = list(self.parents)
+        if not include_bucket:
+            parents.pop(0)
+        path = "/".join(parents)
+        if path:
+            path += "/"
+        path += self.name
+        if include_remote:
+            return f"{self.remote}{path}"
+        return path
+
+
+def parse_file(file_path: str) -> FilePathParts:
+    """Parse file path into parts."""
+    assert not file_path.endswith("/"), "This looks like a directory path"
+    parts = file_path.split(":")
+    remote = parts[0]
+    path = parts[1]
+    if path.startswith("/"):
+        path = path[1:]
+    parents = path.split("/")
+    if len(parents) == 1:
+        return FilePathParts(remote=remote, parents=[], name=parents[0])
+    name = parents.pop()
+    return FilePathParts(remote=remote, parents=parents, name=name)
+
+
+class TreeNode:
+    def __init__(
+        self,
+        name: str,
+        child_nodes: dict[str, "TreeNode"] | None = None,
+        files: list[str] | None = None,
+        parent: "TreeNode | None" = None,
+    ):
+        self.name = name
+        self.child_nodes = child_nodes or {}
+        self.files = files or []
+        self.count = 0
+        self.parent = parent
+
+    def add_count_bubble_up(self):
+        self.count += 1
+        if self.parent:
+            self.parent.add_count_bubble_up()
+
+    def get_path(self) -> str:
+        paths_reversed: list[str] = [self.name]
+        node: TreeNode | None = self
+        assert node is not None
+        while True:
+            node = node.parent
+            if node is None:
+                break
+            paths_reversed.append(node.name)
+        return "/".join(reversed(paths_reversed))
+
+    def get_child_subpaths(self, parent_path: str | None = None) -> list[str]:
+        paths: list[str] = []
+        for child in self.child_nodes.values():
+            child_paths = child.get_child_subpaths(parent_path=child.name)
+            paths.extend(child_paths)
+        for file in self.files:
+            if parent_path:
+                file = f"{parent_path}/{file}"
+            paths.append(file)
+        return paths
+
+    def __repr__(self, indent: int = 0) -> str:
+        # return f"{self.name}: {self.count}, {len(self.children)}"
+        leftpad = " " * indent
+        msg = f"{leftpad}{self.name}: {self.count}"
+        if self.child_nodes:
+            # msg += f"\n   {len(self.children)} children"
+            msg += "\n"
+            for child in self.child_nodes.values():
+                if isinstance(child, TreeNode):
+                    msg += child.__repr__(indent + 2)
+                else:
+                    msg += f"{leftpad}  {child}\n"
+        return msg
+
+
+def _merge(node: TreeNode, parent_path: str, out: dict[str, list[str]]) -> None:
+    parent_path = parent_path + "/" + node.name
+    if not node.child_nodes and not node.files:
+        return  # done
+    if node.files:
+        # we saw files, to don't try to go any deeper.
+        filelist = out.setdefault(parent_path, [])
+        paths = node.get_child_subpaths()
+        for path in paths:
+            filelist.append(path)
+        out[parent_path] = filelist
+        return
+
+    n_child_nodes = len(node.child_nodes)
+
+    if n_child_nodes <= 2:
+        for child in node.child_nodes.values():
+            _merge(child, parent_path, out)
+        return
+
+    filelist = out.setdefault(parent_path, [])
+    paths = node.get_child_subpaths()
+    for path in paths:
+        filelist.append(path)
+    out[parent_path] = filelist
+    return
+
+
+def _make_tree(files: list[str]) -> dict[str, TreeNode]:
+    tree: dict[str, TreeNode] = {}
+    for file in files:
+        parts = parse_file(file)
+        remote = parts.remote
+        node: TreeNode = tree.setdefault(remote, TreeNode(remote))
+        if parts.parents:
+            for parent in parts.parents:
+                is_last = parent == parts.parents[-1]
+                node = node.child_nodes.setdefault(
+                    parent, TreeNode(parent, parent=node)
+                )
+                if is_last:
+                    node.files.append(parts.name)
+                    node.add_count_bubble_up()
+        else:
+            node.files.append(parts.name)
+            node.add_count_bubble_up()
+
+    return tree
+
+
+#
+def _fixup_rclone_paths(outpaths: dict[str, list[str]]) -> dict[str, list[str]]:
+    out: dict[str, list[str]] = {}
+    for path, files in outpaths.items():
+        # fixup path
+        assert path.startswith("/"), "Path should start with /"
+        path = path[1:]
+        # replace the first / with :
+        path = path.replace("/", ":", 1)
+        out[path] = files
+    return out
+
+
+def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
+    if fully_qualified is False:
+        for i, file in enumerate(files):
+            file = "root:" + file
+            files[i] = file
+    tree: dict[str, TreeNode] = _make_tree(files)
+    outpaths: dict[str, list[str]] = {}
+    for _, node in tree.items():
+        _merge(node, "", outpaths)
+    tmp: dict[str, list[str]] = _fixup_rclone_paths(outpaths=outpaths)
+    out: dict[str, list[str]] = {}
+    if fully_qualified is False:
+        for path, files in tmp.items():
+            if path.startswith("root"):
+                path = path.replace("root", "")
+                if path.startswith(":"):
+                    path = path[1:]
+            out[path] = [file.replace("/root/", "") for file in files]
+    else:
+        out = tmp
+    return out
+
+
+def group_under_remote_bucket(
+    files: list[str], fully_qualified: bool = True
+) -> dict[str, list[str]]:
+    """split between filename and bucket"""
+    assert fully_qualified is True, "Not implemented for fully_qualified=False"
+    out: dict[str, list[str]] = {}
+    for file in files:
+        parsed = parse_file(file)
+        remote = f"{parsed.remote}:"
+        parts = parsed.parents
+        bucket = parts[0]
+        remote_bucket = f"{remote}{bucket}"
+        file_list = out.setdefault(remote_bucket, [])
+        file_list.append(parsed.to_string(include_remote=False, include_bucket=False))
+    return out
+
+
+def _get_prefix(path: str) -> tuple[str, str] | None:
+    path_path: Path = Path(path)
+    parts = path_path.parts
+    if len(parts) == 1:
+        return None
+    return parts[0], "/".join(parts[1:])
+
+
+def _common_prefix(prefix: str, files: list[str]) -> PrefixResult:
+    if not files:
+        return PrefixResult(prefix=prefix, files=[])
+    prefix = prefix
+    tmp: list[str] = list(files)
+    while True:
+        if not tmp:
+            break
+        prefix_set: set[str | None] = set()
+        for file in tmp:
+            pair = _get_prefix(file)
+            if pair is None:
+                break
+            _prefix, _ = pair
+            prefix_set.add(_prefix)
+        if len(prefix_set) > 1 or len(prefix_set) == 0:
+            break
+        next_prefix: str | None = prefix_set.pop()
+        if next_prefix is None:
+            break
+        prefix += f"/{next_prefix}"
+        new_tmp: list[str] = []
+        for file in tmp:
+            pair = _get_prefix(file)
+            assert pair is not None
+            _, path = pair
+            new_tmp.append(path)
+        tmp = new_tmp
+    return PrefixResult(prefix=prefix, files=tmp)
+
+
+def group_under_one_prefix(prefix: str, files: list[str]) -> tuple[str, list[str]]:
+    """Group files under one prefix."""
+    if not files:
+        return prefix, []
+    prefix = prefix
+    result = _common_prefix(prefix, files)
+    return result.prefix.replace(":/", ":"), result.files
+
+
+__all__ = ["group_files", "group_under_remote_bucket", "group_under_one_prefix"]