cloud-files 4.24.2__tar.gz → 4.30.0__tar.gz

{cloud-files-4.24.2 → cloud_files-4.30.0}/AUTHORS

@@ -1,6 +1,7 @@
 Manuel Castro <macastro@princeton.edu>
 Nico Kemnitz <nkemnitz@princeton.edu>
 V24 <55334829+umarfarouk98@users.noreply.github.com>
+William Silversmith <william.silvermsith@gmail.com>
 William Silversmith <william.silversmith@gmail.com>
 madiganz <madiganz@users.noreply.github.com>
 ranlu <ranlu@users.noreply.github.com>

{cloud-files-4.24.2 → cloud_files-4.30.0}/ChangeLog

@@ -1,6 +1,43 @@
 CHANGES
 =======
 
+4.30.0
+------
+
+* redesign: normalize cloudpaths so file:// isn't required
+
+4.29.0
+------
+
+* feat(size): add return\_sum argument
+
+4.28.1
+------
+
+* fix(CloudFile.join): add definition of join to CloudFile
+
+4.28.0
+------
+
+* feat: add cf.move(s), cf.touch methods and cli mv, touch commands (#107)
+* fix: add drop table stats
+* perf: add in stats table for faster xfer performance
+* feat: import improvements to ResumableFileSet from transcoder
+* fix: release in xfer
+* fix: leasing was broken
+* feat: add middleauth+https paths indicate CAVE interface (#106)
+
+4.26.0
+------
+
+* feat: make it possible to normalize e.g. zarr2://./helloworld
+* feat: add all current neuroglancer formats to parsing list
+
+4.25.0
+------
+
+* feat: list for apache servers (#104)
+
 4.24.2
 ------
 

{cloud-files-4.24.2 → cloud_files-4.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.30.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
@@ -88,6 +88,12 @@ cf.delete(paths, parallel=2) # threaded + two processes
 boolean = cf.exists('filename')
 results = cf.exists([ 'filename_1', ... ]) # threaded
 
+cf.move("a", "gs://bucket/b")
+cf.moves("gs://bucket/", [ ("a", "b") ])
+
+cf.touch("example")
+cf.touch([ "example", "example2" ])
+
 # for single files
 cf = CloudFile("gs://bucket/file1")
 info = cf.head()
@@ -464,6 +470,10 @@ cloudfiles -p 2 cp --progress -r s3://bkt/ gs://bkt2/
 cloudfiles cp -c br s3://bkt/file.txt gs://bkt2/
 # decompress
 cloudfiles cp -c none s3://bkt/file.txt gs://bkt2/
+# move or rename files
+cloudfiles mv s3://bkt/file.txt gs://bkt2/
+# create an empty file if not existing
+cloudfiles touch s3://bkt/empty.txt
 # pass from stdin (use "-" for source argument)
 find some_dir | cloudfiles cp - s3://bkt/
 # resumable transfers

{cloud-files-4.24.2 → cloud_files-4.30.0}/README.md

@@ -38,6 +38,12 @@ cf.delete(paths, parallel=2) # threaded + two processes
 boolean = cf.exists('filename')
 results = cf.exists([ 'filename_1', ... ]) # threaded
 
+cf.move("a", "gs://bucket/b")
+cf.moves("gs://bucket/", [ ("a", "b") ])
+
+cf.touch("example")
+cf.touch([ "example", "example2" ])
+
 # for single files
 cf = CloudFile("gs://bucket/file1")
 info = cf.head()
@@ -414,6 +420,10 @@ cloudfiles -p 2 cp --progress -r s3://bkt/ gs://bkt2/
 cloudfiles cp -c br s3://bkt/file.txt gs://bkt2/
 # decompress
 cloudfiles cp -c none s3://bkt/file.txt gs://bkt2/
+# move or rename files
+cloudfiles mv s3://bkt/file.txt gs://bkt2/
+# create an empty file if not existing
+cloudfiles touch s3://bkt/empty.txt
 # pass from stdin (use "-" for source argument)
 find some_dir | cloudfiles cp - s3://bkt/
 # resumable transfers

{cloud-files-4.24.2 → cloud_files-4.30.0}/automated_test.py

@@ -564,6 +564,15 @@ def test_path_extraction():
         'a/username2/b/c/d', None, None
       ))
 
+def test_middleauth_path_extraction():
+  import cloudfiles.paths
+  path = cloudfiles.paths.extract('middleauth+https://example.com/wow/cool/')
+  assert path.format == 'precomputed'
+  assert path.protocol == 'middleauth+https'
+  assert path.bucket is None
+  assert path.path == 'wow/cool/'
+  assert path.host == "https://example.com"
+
 @pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))
 def test_access_non_cannonical_minimal_path(s3, protocol):
   from cloudfiles import CloudFiles, exceptions
@@ -1142,3 +1151,92 @@ def test_lock_clearing():
   assert len(lst) == 0
 
 
+@pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))
+def test_move(s3, protocol):
+  from cloudfiles import CloudFiles
+
+  url = compute_url(protocol, "move")
+
+  cf = CloudFiles(url)
+  cf.puts([
+    ('hello', b'world'),
+    ('lamp', b'emporium'),
+  ])
+  cf.move("hello", f"{url}/hola")
+  
+  assert all(cf.exists(["hola"]).values()) == True
+  assert all(cf.exists(["hello"]).values()) == False
+
+  cf.puts([
+    ('hello', b'world'),
+    ('lamp', b'emporium'),
+  ])
+
+  cf.delete("hola")
+
+  cf.moves(f"{url}", [
+    ("hello", f"hola"),
+    ("lamp", f"lampara"),
+  ])
+
+  assert all(cf.exists(["hola", "lampara"]).values()) == True
+  assert all(cf.exists(["hello", "lamp"]).values()) == False
+
+  cf.delete([ "hola", "hello", "lamp", "lampara" ])
+
+@pytest.mark.parametrize("protocol", ["file", "s3"])
+def test_cli_move_python(s3, protocol):
+  from cloudfiles_cli.cloudfiles_cli import _mv_single
+  from cloudfiles import CloudFiles, exceptions
+
+  test_dir = compute_url(protocol, "cli_mv_python")
+  test_dir2 = compute_url(protocol, "cli_mv_python2")
+  cf = CloudFiles(test_dir)
+
+  N = 100
+
+  def mkfiles():
+    cf.delete(cf.list())
+    for i in range(N):
+      cf[str(i)] = b"hello world"
+
+  def run_mv(src, dest):
+    _mv_single(
+      src, dest, 
+      progress=False, block_size=5,
+      part_bytes=int(100e6), no_sign_request=True,
+      parallel=1
+    )
+
+  mkfiles()
+  run_mv(test_dir, test_dir2)
+  assert sorted(list(cf)) == []
+
+  cf2 = CloudFiles(test_dir2)
+  print(sorted(list(cf2)))
+  assert sorted(list(cf2)) == sorted([ f'{i}' for i in range(N) ])
+
+  mkfiles()
+  run_mv(f"{test_dir}/*", f"{test_dir}/move/")
+  assert sorted(list(cf.list(prefix="move"))) == sorted([ f'move/{i}' for i in range(N) ])
+
+  mkfiles()
+  run_mv(f"{test_dir}/1", f"{test_dir}/move/1")
+  assert cf.exists("move/1") == True
+  assert cf.exists("1") == False
+
+@pytest.mark.parametrize("protocol", ["file", "mem", "s3"])
+def test_touch(s3, protocol):
+  from cloudfiles import CloudFiles
+
+  url = compute_url(protocol, "touch")
+
+  cf = CloudFiles(url)
+
+  cf.touch([ str(i) for i in range(20) ])
+
+  assert sorted(list(cf)) == sorted([ str(i) for i in range(20) ])
+
+  cf.touch([ str(i) for i in range(20) ])
+
+  assert sorted(list(cf)) == sorted([ str(i) for i in range(20) ])

{cloud-files-4.24.2 → cloud_files-4.30.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.30.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
@@ -88,6 +88,12 @@ cf.delete(paths, parallel=2) # threaded + two processes
 boolean = cf.exists('filename')
 results = cf.exists([ 'filename_1', ... ]) # threaded
 
+cf.move("a", "gs://bucket/b")
+cf.moves("gs://bucket/", [ ("a", "b") ])
+
+cf.touch("example")
+cf.touch([ "example", "example2" ])
+
 # for single files
 cf = CloudFile("gs://bucket/file1")
 info = cf.head()
@@ -464,6 +470,10 @@ cloudfiles -p 2 cp --progress -r s3://bkt/ gs://bkt2/
 cloudfiles cp -c br s3://bkt/file.txt gs://bkt2/
 # decompress
 cloudfiles cp -c none s3://bkt/file.txt gs://bkt2/
+# move or rename files
+cloudfiles mv s3://bkt/file.txt gs://bkt2/
+# create an empty file if not existing
+cloudfiles touch s3://bkt/empty.txt
 # pass from stdin (use "-" for source argument)
 find some_dir | cloudfiles cp - s3://bkt/
 # resumable transfers

cloud_files-4.30.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "97a23a5", "is_release": true}

{cloud-files-4.24.2 → cloud_files-4.30.0}/cloudfiles/cloudfiles.py

@@ -2,7 +2,7 @@ from typing import (
   Any, Dict, Optional, 
   Union, List, Tuple, 
   Callable, Generator, 
-  Iterable, cast, BinaryIO
+  Sequence, cast, BinaryIO
 )
 
 from queue import Queue
@@ -29,10 +29,10 @@ from . import compression, paths, gcs
 from .exceptions import UnsupportedProtocolError, MD5IntegrityError, CRC32CIntegrityError
 from .lib import (
   mkdir, totalfn, toiter, scatter, jsonify, nvl, 
-  duplicates, first, sip,
+  duplicates, first, sip, touch,
   md5, crc32c, decode_crc32c_b64
 )
-from .paths import ALIASES
+from .paths import ALIASES, find_common_buckets
 from .secrets import CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
 from .threaded_queue import ThreadedQueue, DEFAULT_THREADS
 from .typing import (
@@ -44,7 +44,7 @@ from .scheduler import schedule_jobs
 from .interfaces import (
   FileInterface, HttpInterface, 
   S3Interface, GoogleCloudStorageInterface,
-  MemoryInterface
+  MemoryInterface, CaveInterface,
 )
 
 INTERFACES = {
@@ -54,6 +54,7 @@ INTERFACES = {
   'http': HttpInterface,
   'https': HttpInterface,
   'mem': MemoryInterface,
+  'middleauth+https': CaveInterface,
 }
 for alias in ALIASES:
   INTERFACES[alias] = S3Interface
@@ -181,7 +182,7 @@ def path_to_byte_range_tags(path):
   if isinstance(path, str):
     return (path, None, None, None)
   return (path['path'], path.get('start', None), path.get('end', None), path.get('tags', None))
-  
+
 def dl(
   cloudpaths:GetPathType, raw:bool=False, **kwargs
 ) -> Union[bytes,List[dict]]:
@@ -192,23 +193,8 @@ def dl(
   dict.
   """
   cloudpaths, is_multiple = toiter(cloudpaths, is_iter=True)
-  clustered = defaultdict(list)
-  total = 0
-  for path in cloudpaths:
-    pth = path
-    byte_range = None
-    if isinstance(path, dict):
-      pth = path["path"]
-      byte_range = path["byte_range"]
-
-    epath = paths.extract(pth)
-    bucketpath = paths.asbucketpath(epath)
-    clustered[bucketpath].append({ 
-      "path": epath.path,
-      "start": (byte_range[0] if byte_range else None), # type: ignore
-      "end": (byte_range[1] if byte_range else None), # type: ignore
-    })
-    total += 1
+  clustered = find_common_buckets(cloudpaths)
+  total = sum([ len(bucket) for bucket in clustered.values() ])
 
   progress = kwargs.get("progress", False) and total > 1
   pbar = tqdm(total=total, desc="Downloading", disable=(not progress))
@@ -256,6 +242,8 @@ class CloudFiles:
     if use_https:
       cloudpath = paths.to_https_protocol(cloudpath)
 
+    cloudpath = paths.normalize(cloudpath)
+
     self.cloudpath = cloudpath
     self.progress = progress
     self.secrets = secrets
@@ -851,8 +839,10 @@ class CloudFiles:
 
   def size(
     self, paths:GetPathType, 
-    total:Optional[int] = None, progress:Optional[bool] = None
-  ) -> Union[Dict[str,int],List[Dict[str,int]]]:
+    total:Optional[int] = None, 
+    progress:Optional[bool] = None,
+    return_sum:bool = False,
+  ) -> Union[Dict[str,int],List[Dict[str,int]],int]:
     """
     Get the size in bytes of one or more files in its stored state.
     """
@@ -873,6 +863,9 @@ class CloudFiles:
       green=self.green,
     )
 
+    if return_sum:
+      return sum(( sz for sz in results.values() ))
+
     if return_multiple:
       return results
     return first(results.values())
@@ -918,6 +911,60 @@ class CloudFiles:
     )
     return len(results)
 
+  def touch(
+    self, 
+    paths:GetPathType,
+    progress:Optional[bool] = None,
+    total:Optional[int] = None,
+    nocopy:bool = False,
+  ):
+    """
+    Create a zero byte file if it doesn't exist.
+    """
+    paths = toiter(paths)
+    progress = nvl(progress, self.progress)
+    total = totalfn(paths, total)
+
+    if self.protocol == "file":
+      basepath = self.cloudpath.replace("file://", "")
+      for path in tqdm(paths, disable=(not progress), total=total):
+        pth = path
+        if isinstance(path, dict):
+          pth = path["path"]
+        touch(self.join(basepath, pth))
+      return
+
+    results = self.exists(paths, total=total, progress=progress)
+
+    dne = [ 
+      (fname, b'') 
+      for fname, exists in results.items() 
+      if not exists 
+    ]
+
+    self.puts(dne, progress=progress)
+
+    # def thunk_copy(path):
+    #   with self._get_connection() as conn:
+    #     conn.copy_file(path, self._path.bucket, self.join(self._path.path, path))
+    #   return 1
+
+    # if not nocopy:
+    #   already_exists = ( 
+    #     fname
+    #     for fname, exists in results.items() 
+    #     if exists 
+    #   )
+
+    #   results = schedule_jobs(
+    #     fns=( partial(thunk_copy, path) for path in already_exists ),
+    #     progress=progress,
+    #     total=(total - len(dne)),
+    #     concurrency=self.num_threads,
+    #     green=self.green,
+    #     count_return=True,
+    #   )
+
   def list(
     self, prefix:str = "", flat:bool = False
   ) -> Generator[str,None,None]:
@@ -952,6 +999,7 @@ class CloudFiles:
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
     allow_missing:bool = False,
+    progress:Optional[bool] = None,
   ) -> None:
     """
     Transfer all files from this CloudFiles storage 
@@ -968,7 +1016,7 @@ class CloudFiles:
       - gs->gs: Uses GCS copy API to minimize data movement
       - s3->s3: Uses boto s3 copy API to minimize data movement
 
-    cf_src: another CloudFiles instance or cloudpath 
+    cf_dest: another CloudFiles instance or cloudpath 
     paths: if None transfer all files from src, else if
       an iterable, transfer only these files.
 
@@ -996,7 +1044,8 @@ class CloudFiles:
     return cf_dest.transfer_from(
       self, paths, block_size, 
       reencode, content_type,
-      allow_missing,
+      allow_missing, 
+      progress,
     )
 
   def transfer_from(
@@ -1007,6 +1056,7 @@ class CloudFiles:
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
     allow_missing:bool = False,
+    progress:Optional[bool] = None,
   ) -> None:
     """
     Transfer all files from the source CloudFiles storage 
@@ -1053,7 +1103,15 @@ class CloudFiles:
 
     total = totalfn(paths, None)
 
-    with tqdm(desc="Transferring", total=total, disable=(not self.progress)) as pbar:
+    disable = progress
+    if disable is None:
+      disable = self.progress
+    if disable is None:
+      disable = False
+    else:
+      disable = not disable
+
+    with tqdm(desc="Transferring", total=total, disable=disable) as pbar:
       if (
         cf_src.protocol == "file"
         and self.protocol == "file"
@@ -1210,6 +1268,9 @@ class CloudFiles:
           else:
             raise
 
+        if dest_path == '':
+          dest_path = src_path
+
         to_upload.append({
           "path": dest_path,
           "content": handle,
@@ -1261,6 +1322,99 @@ class CloudFiles:
     )
     return len(results)
 
+  def move(self, src:str, dest:str):
+    """Move (rename) src to dest.
+
+    src and dest do not have to be on the same filesystem.
+    """
+    epath = paths.extract(dest)
+    full_cloudpath = paths.asprotocolpath(epath)
+    dest_cloudpath = paths.dirname(full_cloudpath)
+    base_dest = paths.basename(full_cloudpath)
+
+    return self.moves(dest_cloudpath, [
+      (src, base_dest)
+    ], block_size=1, progress=False)
+
+  def moves(
+    self,
+    cf_dest:Any,
+    paths:Union[Sequence[str], Sequence[Tuple[str, str]]],
+    block_size:int = 64, 
+    total:Optional[int] = None,
+    progress:Optional[bool] = None,
+  ):
+    """
+    Move (rename) files.
+
+    pairs: [ (src, dest), (src, dest), ... ]
+    """
+    if isinstance(cf_dest, str):
+      cf_dest = CloudFiles(
+        cf_dest, progress=False, 
+        green=self.green, num_threads=self.num_threads,
+      )
+    
+    total = totalfn(paths, total)
+
+    disable = not (self.progress if progress is None else progress)
+
+    if self.protocol == "file" and cf_dest.protocol == "file":
+      self.__moves_file_to_file(
+        cf_dest, paths, total, 
+        disable, block_size
+      )
+      return
+
+    pbar = tqdm(total=total, disable=disable, desc="Moving")
+
+    with pbar:
+      for subpairs in sip(paths, block_size):
+        subpairs = [
+          ((pair, pair) if isinstance(pair, str) else pair)
+          for pair in subpairs
+        ]
+
+        self.transfer_to(cf_dest, paths=(
+          {
+            "path": src,
+            "dest_path": dest,
+          } 
+          for src, dest in subpairs
+        ), progress=False)
+        self.delete(( src for src, dest in subpairs ), progress=False)
+        pbar.update(len(subpairs))
+
+  def __moves_file_to_file(
+    self, 
+    cf_dest:Any, 
+    paths:Union[Sequence[str], Sequence[Tuple[str,str]]],
+    total:Optional[int], 
+    disable:bool,
+    block_size:int,
+  ):
+    for pair in tqdm(paths, total=total, disable=disable, desc="Moving"):
+      if isinstance(pair, str):
+        src = pair
+        dest = pair
+      else:
+        (src, dest) = pair
+
+      src = self.join(self.cloudpath, src).replace("file://", "")
+      dest = cf_dest.join(cf_dest.cloudpath, dest).replace("file://", "")
+
+      if os.path.isdir(dest):
+        dest = cf_dest.join(dest, os.path.basename(src))
+      else:
+        mkdir(os.path.dirname(dest))
+
+      src, encoding = FileInterface.get_encoded_file_path(src)
+      _, dest_ext = os.path.splitext(dest)
+      dest_ext_compress = FileInterface.get_extension(encoding)
+      if dest_ext_compress != dest_ext:
+        dest += dest_ext_compress
+      shutil.move(src, dest)
+
   def join(self, *paths:str) -> str:
     """
     Convenience method for joining path strings
@@ -1439,6 +1593,16 @@ class CloudFile:
       reencode=reencode,
     )
 
+  def join(self, *args):
+    return self.cf.join(*args)
+
+  def touch(self):
+    return self.cf.touch(self.filename)
+
+  def move(self, dest):
+    """Move (rename) this file to dest."""
+    return self.cf.move(self.filename, dest)
+
   def __len__(self):
     return self.size()