fsspec 2023.6.0__tar.gz → 2023.9.1__tar.gz

{fsspec-2023.6.0/fsspec.egg-info → fsspec-2023.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fsspec
-Version: 2023.6.0
+Version: 2023.9.1
 Summary: File-system specification
 Home-page: http://github.com/fsspec/filesystem_spec
 Maintainer: Martin Durant

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-06-09T13:30:57-0400",
+ "date": "2023-09-15T16:17:21-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "9a1e624022f3ad39071de5b17bafa23214b8662b",
- "version": "2023.6.0"
+ "full-revisionid": "247b249a008990c584d2619f030bd42916a82e4a",
+ "version": "2023.9.1"
 }
 '''  # END VERSION_JSON
 

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/asyn.py

@@ -13,12 +13,7 @@ from typing import TYPE_CHECKING, Iterable
 
 from .callbacks import _DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
-from .implementations.local import (
-    LocalFileSystem,
-    make_path_posix,
-    trailing_sep,
-    trailing_sep_maybe_asterisk,
-)
+from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
 from .spec import AbstractBufferedFile, AbstractFileSystem
 from .utils import is_exception, other_paths
 
@@ -83,6 +78,8 @@ def sync(loop, func, *args, timeout=None, **kwargs):
         loop0 = asyncio.events.get_running_loop()
         if loop0 is loop:
             raise NotImplementedError("Calling sync() from within a running loop")
+    except NotImplementedError:
+        raise
     except RuntimeError:
         pass
     coro = func(*args, **kwargs)
@@ -347,26 +344,42 @@ class AsyncFileSystem(AbstractFileSystem):
         elif on_error is None:
             on_error = "raise"
 
-        source_is_str = isinstance(path1, str)
-        paths = await self._expand_path(path1, maxdepth=maxdepth, recursive=recursive)
-        if source_is_str and (not recursive or maxdepth is not None):
-            # Non-recursive glob does not copy directories
-            paths = [p for p in paths if not (trailing_sep(p) or await self._isdir(p))]
-            if not paths:
-                return
+        if isinstance(path1, list) and isinstance(path2, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            paths1 = path1
+            paths2 = path2
+        else:
+            source_is_str = isinstance(path1, str)
+            paths1 = await self._expand_path(
+                path1, maxdepth=maxdepth, recursive=recursive
+            )
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                paths1 = [
+                    p for p in paths1 if not (trailing_sep(p) or await self._isdir(p))
+                ]
+                if not paths1:
+                    return
+
+            source_is_file = len(paths1) == 1
+            dest_is_dir = isinstance(path2, str) and (
+                trailing_sep(path2) or await self._isdir(path2)
+            )
+
+            exists = source_is_str and (
+                (has_magic(path1) and source_is_file)
+                or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1))
+            )
+            paths2 = other_paths(
+                paths1,
+                path2,
+                exists=exists,
+                flatten=not source_is_str,
+            )
 
-        isdir = isinstance(path2, str) and (
-            trailing_sep(path2) or await self._isdir(path2)
-        )
-        path2 = other_paths(
-            paths,
-            path2,
-            exists=isdir and source_is_str and not trailing_sep_maybe_asterisk(path1),
-            is_dir=isdir,
-            flatten=not source_is_str,
-        )
         batch_size = batch_size or self.batch_size
-        coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths, path2)]
+        coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)]
         result = await _run_coros_in_chunks(
             coros, batch_size=batch_size, return_exceptions=True, nofiles=True
         )
@@ -501,28 +514,39 @@ class AsyncFileSystem(AbstractFileSystem):
         constructor, or for all instances by setting the "gather_batch_size" key
         in ``fsspec.config.conf``, falling back to 1/8th of the system limit .
         """
-        source_is_str = isinstance(lpath, str)
-        if source_is_str:
-            lpath = make_path_posix(lpath)
-        fs = LocalFileSystem()
-        lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth)
-        if source_is_str and (not recursive or maxdepth is not None):
-            # Non-recursive glob does not copy directories
-            lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))]
-            if not lpaths:
-                return
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            source_is_str = isinstance(lpath, str)
+            if source_is_str:
+                lpath = make_path_posix(lpath)
+            fs = LocalFileSystem()
+            lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth)
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))]
+                if not lpaths:
+                    return
+
+            source_is_file = len(lpaths) == 1
+            dest_is_dir = isinstance(rpath, str) and (
+                trailing_sep(rpath) or await self._isdir(rpath)
+            )
 
-        isdir = isinstance(rpath, str) and (
-            trailing_sep(rpath) or await self._isdir(rpath)
-        )
-        rpath = self._strip_protocol(rpath)
-        rpaths = other_paths(
-            lpaths,
-            rpath,
-            exists=isdir and source_is_str and not trailing_sep_maybe_asterisk(lpath),
-            is_dir=isdir,
-            flatten=not source_is_str,
-        )
+            rpath = self._strip_protocol(rpath)
+            exists = source_is_str and (
+                (has_magic(lpath) and source_is_file)
+                or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath))
+            )
+            rpaths = other_paths(
+                lpaths,
+                rpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
 
         is_dir = {l: os.path.isdir(l) for l in lpaths}
         rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]]
@@ -567,32 +591,44 @@ class AsyncFileSystem(AbstractFileSystem):
         constructor, or for all instances by setting the "gather_batch_size" key
         in ``fsspec.config.conf``, falling back to 1/8th of the system limit .
         """
-        source_is_str = isinstance(rpath, str)
-        # First check for rpath trailing slash as _strip_protocol removes it.
-        source_not_trailing_sep = source_is_str and not trailing_sep_maybe_asterisk(
-            rpath
-        )
-        rpath = self._strip_protocol(rpath)
-        rpaths = await self._expand_path(rpath, recursive=recursive)
-        if source_is_str and (not recursive or maxdepth is not None):
-            # Non-recursive glob does not copy directories
-            rpaths = [
-                p for p in rpaths if not (trailing_sep(p) or await self._isdir(p))
-            ]
-            if not rpaths:
-                return
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            source_is_str = isinstance(rpath, str)
+            # First check for rpath trailing slash as _strip_protocol removes it.
+            source_not_trailing_sep = source_is_str and not trailing_sep(rpath)
+            rpath = self._strip_protocol(rpath)
+            rpaths = await self._expand_path(
+                rpath, recursive=recursive, maxdepth=maxdepth
+            )
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                rpaths = [
+                    p for p in rpaths if not (trailing_sep(p) or await self._isdir(p))
+                ]
+                if not rpaths:
+                    return
+
+            lpath = make_path_posix(lpath)
+            source_is_file = len(rpaths) == 1
+            dest_is_dir = isinstance(lpath, str) and (
+                trailing_sep(lpath) or LocalFileSystem().isdir(lpath)
+            )
+
+            exists = source_is_str and (
+                (has_magic(rpath) and source_is_file)
+                or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep)
+            )
+            lpaths = other_paths(
+                rpaths,
+                lpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
 
-        lpath = make_path_posix(lpath)
-        isdir = isinstance(lpath, str) and (
-            trailing_sep(lpath) or LocalFileSystem().isdir(lpath)
-        )
-        lpaths = other_paths(
-            rpaths,
-            lpath,
-            exists=isdir and source_not_trailing_sep,
-            is_dir=isdir,
-            flatten=not source_is_str,
-        )
         [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths]
         batch_size = kwargs.pop("batch_size", self.batch_size)
 
@@ -639,7 +675,7 @@ class AsyncFileSystem(AbstractFileSystem):
     async def _ls(self, path, detail=True, **kwargs):
         raise NotImplementedError
 
-    async def _walk(self, path, maxdepth=None, **kwargs):
+    async def _walk(self, path, maxdepth=None, on_error="omit", **kwargs):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
@@ -651,7 +687,11 @@ class AsyncFileSystem(AbstractFileSystem):
         detail = kwargs.pop("detail", False)
         try:
             listing = await self._ls(path, detail=True, **kwargs)
-        except (FileNotFoundError, OSError):
+        except (FileNotFoundError, OSError) as e:
+            if on_error == "raise":
+                raise
+            elif callable(on_error):
+                on_error(e)
             if detail:
                 yield path, {}, {}
             else:
@@ -689,25 +729,24 @@ class AsyncFileSystem(AbstractFileSystem):
             ):
                 yield _
 
-    async def _glob(self, path, **kwargs):
+    async def _glob(self, path, maxdepth=None, **kwargs):
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
         import re
 
         ends = path.endswith("/")
         path = self._strip_protocol(path)
-        indstar = path.find("*") if path.find("*") >= 0 else len(path)
-        indques = path.find("?") if path.find("?") >= 0 else len(path)
-        indbrace = path.find("[") if path.find("[") >= 0 else len(path)
+        idx_star = path.find("*") if path.find("*") >= 0 else len(path)
+        idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
+        idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
 
-        ind = min(indstar, indques, indbrace)
+        min_idx = min(idx_star, idx_qmark, idx_brace)
 
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            root = path
-            depth = 1
-            if ends:
-                path += "/*"
-            elif await self._exists(path):
+            if await self._exists(path):
                 if not detail:
                     return [path]
                 else:
@@ -717,13 +756,21 @@ class AsyncFileSystem(AbstractFileSystem):
                     return []  # glob of non-existent returns empty
                 else:
                     return {}
-        elif "/" in path[:ind]:
-            ind2 = path[:ind].rindex("/")
-            root = path[: ind2 + 1]
-            depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1
+        elif "/" in path[:min_idx]:
+            min_idx = path[:min_idx].rindex("/")
+            root = path[: min_idx + 1]
+            depth = path[min_idx + 1 :].count("/") + 1
         else:
             root = ""
-            depth = None if "**" in path else path[ind + 1 :].count("/") + 1
+            depth = path[min_idx + 1 :].count("/") + 1
+
+        if "**" in path:
+            if maxdepth is not None:
+                idx_double_stars = path.find("**")
+                depth_double_stars = path[idx_double_stars:].count("/") + 1
+                depth = depth - depth_double_stars + maxdepth
+            else:
+                depth = None
 
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
@@ -751,14 +798,23 @@ class AsyncFileSystem(AbstractFileSystem):
             )
             + "$"
         )
-        pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern)
+        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
+        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
         pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*"))
+        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
+        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+        pattern = re.compile(pattern)
         out = {
             p: allpaths[p]
             for p in sorted(allpaths)
             if pattern.match(p.replace("//", "/").rstrip("/"))
         }
+
+        # Return directories only when the glob end by a slash
+        # This is needed for posix glob compliance
+        if ends:
+            out = {k: v for k, v in out.items() if v["type"] == "directory"}
+
         if detail:
             return out
         else:
@@ -777,8 +833,14 @@ class AsyncFileSystem(AbstractFileSystem):
 
     async def _find(self, path, maxdepth=None, withdirs=False, **kwargs):
         path = self._strip_protocol(path)
-        out = dict()
+        out = {}
         detail = kwargs.pop("detail", False)
+
+        # Add the root directory if withdirs is requested
+        # This is needed for posix glob compliance
+        if withdirs and path != "" and await self._isdir(path):
+            out[path] = await self._info(path)
+
         # async for?
         async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs):
             if withdirs:
@@ -805,7 +867,7 @@ class AsyncFileSystem(AbstractFileSystem):
             path = [self._strip_protocol(p) for p in path]
             for p in path:  # can gather here
                 if has_magic(p):
-                    bit = set(await self._glob(p))
+                    bit = set(await self._glob(p, maxdepth=maxdepth))
                     out |= bit
                     if recursive:
                         # glob call above expanded one depth so if maxdepth is defined
@@ -829,7 +891,7 @@ class AsyncFileSystem(AbstractFileSystem):
                     out.add(p)
         if not out:
             raise FileNotFoundError(path)
-        return list(sorted(out))
+        return sorted(out)
 
     async def _mkdir(self, path, create_parents=True, **kwargs):
         pass  # not necessary to implement, may not have directories

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/caching.py

@@ -456,7 +456,7 @@ class KnownPartsOfAFile(BaseCache):
 
         # simple consolidation of contiguous blocks
         if data:
-            old_offsets = sorted(list(data.keys()))
+            old_offsets = sorted(data.keys())
             offsets = [old_offsets[0]]
             blocks = [data.pop(old_offsets[0])]
             for start, stop in old_offsets[1:]:

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/compression.py

@@ -1,5 +1,4 @@
 """Helper functions for a standard streaming compression API"""
-from bz2 import BZ2File
 from zipfile import ZipFile
 
 import fsspec.utils
@@ -68,7 +67,13 @@ def unzip(infile, mode="rb", filename=None, **kwargs):
 
 
 register_compression("zip", unzip, "zip")
-register_compression("bz2", BZ2File, "bz2")
+
+try:
+    from bz2 import BZ2File
+except ImportError:
+    pass
+else:
+    register_compression("bz2", BZ2File, "bz2")
 
 try:  # pragma: no cover
     from isal import igzip

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/core.py

@@ -210,7 +210,6 @@ def open_files(
     num=1,
     protocol=None,
     newline=None,
-    auto_mkdir=True,
     expand=True,
     **kwargs,
 ):
@@ -249,9 +248,6 @@ def open_files(
     newline: bytes or None
         Used for line terminator in text mode. If None, uses system default;
         if blank, uses no translation.
-    auto_mkdir: bool (True)
-        If in write mode, this will ensure the target directory exists before
-        writing, by calling ``fs.mkdirs(exist_ok=True)``.
     expand: bool
     **kwargs: dict
         Extra options that make sense to a particular storage connection, e.g.
@@ -288,9 +284,6 @@ def open_files(
         protocol=protocol,
         expand=expand,
     )
-    if "r" not in mode and auto_mkdir:
-        parents = {fs._parent(path) for path in paths}
-        [fs.makedirs(parent, exist_ok=True) for parent in parents]
     return OpenFiles(
         [
             OpenFile(
@@ -360,6 +353,19 @@ def url_to_fs(url, **kwargs):
     urlpath : str
         The file-systems-specific URL for ``url``.
     """
+    # non-FS arguments that appear in fsspec.open()
+    # inspect could keep this in sync with open()'s signature
+    known_kwargs = {
+        "compression",
+        "encoding",
+        "errors",
+        "expand",
+        "mode",
+        "name_function",
+        "newline",
+        "num",
+    }
+    kwargs = {k: v for k, v in kwargs.items() if k not in known_kwargs}
     chain = _un_chain(url, kwargs)
     inkwargs = {}
     # Reverse iterate the chain, creating a nested target_* structure
@@ -611,7 +617,7 @@ def get_fs_token_paths(
         pchains = [
             _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath
         ]
-        if len(set(pc[1] for pc in pchains)) > 1:
+        if len({pc[1] for pc in pchains}) > 1:
             raise ValueError("Protocol mismatch getting fs from %s", urlpath)
         paths = [pc[0] for pc in pchains]
     else:
@@ -621,6 +627,8 @@ def get_fs_token_paths(
     else:
         if "w" in mode and expand:
             paths = _expand_paths(paths, name_function, num)
+        elif "x" in mode and expand:
+            paths = _expand_paths(paths, name_function, num)
         elif "*" in paths:
             paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)]
         else:

{fsspec-2023.6.0 → fsspec-2023.9.1}/fsspec/generic.py

@@ -1,9 +1,15 @@
+from __future__ import annotations
+
 import inspect
 import logging
+import os
+import shutil
+import uuid
+from typing import Optional
 
-from .asyn import AsyncFileSystem
+from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
 from .callbacks import _DEFAULT_CALLBACK
-from .core import filesystem, get_filesystem_class, split_protocol
+from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs
 
 _generic_fs = {}
 logger = logging.getLogger("fsspec.generic")
@@ -29,7 +35,8 @@ def _resolve_fs(url, method=None, protocol=None, storage_options=None):
         cls = get_filesystem_class(protocol)
         return cls.current()
     if method == "options":
-        return filesystem(protocol, **storage_options.get(protocol, {}))
+        fs, _ = url_to_fs(url, **storage_options.get(protocol, {}))
+        return fs
     raise ValueError(f"Unknown FS resolution method: {method}")
 
 
@@ -51,19 +58,23 @@ def rsync(
     Parameters
     ----------
     source: str
-        Root of the directory tree to take files from.
+        Root of the directory tree to take files from. This must be a directory, but
+        do not include any terminating "/" character
     destination: str
         Root path to copy into. The contents of this location should be
-        identical to the contents of ``source`` when done.
+        identical to the contents of ``source`` when done. This will be made a
+        directory, and the terminal "/" should not be included.
     delete_missing: bool
         If there are paths in the destination that don't exist in the
         source and this is True, delete them. Otherwise, leave them alone.
-    source_field: str
+    source_field: str | callable
         If ``update_field`` is "different", this is the key in the info
-        of source files to consider for difference.
-    dest_field: str
+        of source files to consider for difference. Maybe a function of the
+        info dict.
+    dest_field: str | callable
         If ``update_field`` is "different", this is the key in the info
-        of destination files to consider for difference.
+        of destination files to consider for difference. May be a function of
+        the info dict.
     update_cond: "different"|"always"|"never"
         If "always", every file is copied, regardless of whether it exists in
         the destination. If "never", files that exist in the destination are
@@ -90,9 +101,10 @@ def rsync(
         if v["type"] == "directory" and a.replace(source, destination) not in otherfiles
     ]
     logger.debug(f"{len(dirs)} directories to create")
-    for dirn in dirs:
-        # no async
-        fs.mkdirs(dirn.replace(source, destination), exist_ok=True)
+    if dirs:
+        fs.make_many_dirs(
+            [dirn.replace(source, destination) for dirn in dirs], exist_ok=True
+        )
     allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"}
     logger.debug(f"{len(allfiles)} files to consider for copy")
     to_delete = [
@@ -106,7 +118,10 @@ def rsync(
             if update_cond == "always":
                 allfiles[k] = otherfile
             elif update_cond == "different":
-                if v[source_field] != otherfiles[otherfile][dest_field]:
+                inf1 = source_field(v) if callable(source_field) else v[source_field]
+                v2 = otherfiles[otherfile]
+                inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field]
+                if inf1 != inf2:
                     # details mismatch, make copy
                     allfiles[k] = otherfile
                 else:
@@ -115,12 +130,12 @@ def rsync(
         else:
             # file not in target yet
             allfiles[k] = otherfile
+    logger.debug(f"{len(allfiles)} files to copy")
     if allfiles:
         source_files, target_files = zip(*allfiles.items())
-        logger.debug(f"{len(source_files)} files to copy")
         fs.cp(source_files, target_files, **kwargs)
+    logger.debug(f"{len(to_delete)} files to delete")
     if delete_missing:
-        logger.debug(f"{len(to_delete)} files to delete")
         fs.rm(to_delete)
 
 
@@ -165,11 +180,11 @@ class GenericFileSystem(AsyncFileSystem):
         fs = _resolve_fs(path, self.method)
         if fs.async_impl:
             out = await fs._find(
-                path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs
+                path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs
             )
         else:
             out = fs.find(
-                path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs
+                path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs
             )
         result = {}
         for k, v in out.items():
@@ -238,6 +253,7 @@ class GenericFileSystem(AsyncFileSystem):
             fs.rm(url, **kwargs)
 
     async def _makedirs(self, path, exist_ok=False):
+        logger.debug("Make dir %s", path)
         fs = _resolve_fs(path, self.method)
         if fs.async_impl:
             await fs._makedirs(path, exist_ok=exist_ok)
@@ -294,6 +310,84 @@ class GenericFileSystem(AsyncFileSystem):
                 # fail while opening f1 or f2
                 pass
 
+    async def _make_many_dirs(self, urls, exist_ok=True):
+        fs = _resolve_fs(urls[0], self.method)
+        if fs.async_impl:
+            coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls]
+            await _run_coros_in_chunks(coros)
+        else:
+            for u in urls:
+                fs.makedirs(u, exist_ok=exist_ok)
+
+    make_many_dirs = sync_wrapper(_make_many_dirs)
+
+    async def _copy(
+        self,
+        path1: list[str],
+        path2: list[str],
+        recursive: bool = False,
+        on_error: str = "ignore",
+        maxdepth: Optional[int] = None,
+        batch_size: Optional[int] = None,
+        tempdir: Optional[str] = None,
+        **kwargs,
+    ):
+        if recursive:
+            raise NotImplementedError
+        fs = _resolve_fs(path1[0], self.method)
+        fs2 = _resolve_fs(path2[0], self.method)
+        # not expanding paths atm., assume call is from rsync()
+        if fs is fs2:
+            # pure remote
+            if fs.async_impl:
+                return await fs._copy(path1, path2, **kwargs)
+            else:
+                return fs.copy(path1, path2, **kwargs)
+        await copy_file_op(
+            fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error
+        )
+
+
+async def copy_file_op(
+    fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore"
+):
+    import tempfile
+
+    tempdir = tempdir or tempfile.mkdtemp()
+    try:
+        coros = [
+            _copy_file_op(
+                fs1,
+                u1,
+                fs2,
+                u2,
+                os.path.join(tempdir, uuid.uuid4().hex),
+                on_error=on_error,
+            )
+            for u1, u2 in zip(url1, url2)
+        ]
+        await _run_coros_in_chunks(coros, batch_size=batch_size)
+    finally:
+        shutil.rmtree(tempdir)
+
+
+async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"):
+    ex = () if on_error == "raise" else Exception
+    logger.debug("Copy %s -> %s", url1, url2)
+    try:
+        if fs1.async_impl:
+            await fs1._get_file(url1, local)
+        else:
+            fs1.get_file(url1, local)
+        if fs2.async_impl:
+            await fs2._put_file(local, url2)
+        else:
+            fs2.put_file(local, url2)
+        os.unlink(local)
+        logger.debug("Copy %s -> %s; done", url1, url2)
+    except ex as e:
+        logger.debug("ignoring cp exception for %s: %s", url1, e)
+
 
 async def maybe_await(cor):
     if inspect.iscoroutine(cor):