cloud-files 5.6.1__tar.gz → 5.8.0__tar.gz

{cloud_files-5.6.1 → cloud_files-5.8.0}/ChangeLog

@@ -1,6 +1,17 @@
 CHANGES
 =======
 
+5.8.0
+-----
+
+* docs: add help string for touch
+* feat(cli): add mkdir command
+
+5.7.0
+-----
+
+* feat(cli): support \*.jpg type syntax for ls, cp, head, rm, mv (#118)
+
 5.6.1
 -----
 

{cloud_files-5.6.1 → cloud_files-5.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.6.1
+Version: 5.8.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-5.6.1 → cloud_files-5.8.0}/automated_test.py

@@ -876,6 +876,14 @@ def test_cli_cp():
   subprocess.run(f"cloudfiles cp {destdir}", shell=True)
   assert not os.path.exists(destdir)
 
+  for i in range(N):
+    touch(os.path.join(srcdir, f"{i}jpg"))
+
+  mkdir(destdir)
+  subprocess.run(f"cloudfiles cp {srcdir}/*jpg {destdir}", shell=True)
+  assert len(os.listdir(destdir)) == N
+  shutil.rmtree(destdir)
+
   try:
     shutil.rmtree(srcdir)
   except FileNotFoundError:
@@ -1033,6 +1041,15 @@ def test_cli_rm_shell():
   subprocess.run(["cloudfiles", "rm", "./cloudfiles-deletable-test-file"])
   assert not os.path.exists("./cloudfiles-deletable-test-file")
 
+  mkfiles()
+  for i in range(N):
+    touch(os.path.join(test_dir, f"{i}.jpg"))  
+
+  assert len(os.listdir(test_dir)) == N * 2
+  subprocess.run(["cloudfiles", "rm", test_dir + "/*.jpg"])
+  res = set([ str(_) for _ in range(N) ])
+  assert set(os.listdir(test_dir)) == res
+
   try:
     shutil.rmtree(test_dir)
   except FileNotFoundError:

{cloud_files-5.6.1 → cloud_files-5.8.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.6.1
+Version: 5.8.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud_files-5.8.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "9e3fa7f", "is_release": true}

{cloud_files-5.6.1 → cloud_files-5.8.0}/cloudfiles_cli/cloudfiles_cli.py

@@ -97,7 +97,7 @@ def ls(shortpath, flat, expr, cloudpath, no_auth):
   if no_auth and 's3://' not in cloudpath:
     cloudpath = cloudfiles.paths.to_https_protocol(cloudpath)
 
-  _, flt, prefix = get_mfp(cloudpath, True)
+  _, flt, prefix, suffix = get_mfp(cloudpath, True)
   epath = extract(cloudpath)
   if len(epath.path) > 0:
     if prefix == "" and flt == False:
@@ -128,6 +128,10 @@ def ls(shortpath, flat, expr, cloudpath, no_auth):
     iterables = [ cf.list(prefix=prefix, flat=flat) ]
 
   iterables = itertools.chain(*iterables)
+
+  if suffix:
+    iterables = ( x for x in iterables if x.endswith(suffix) )
+
   for pathset in sip(iterables, 1000):
     if not shortpath:
       pathset = [ cloudpathjoin(cloudpath, pth) for pth in pathset ]
@@ -152,6 +156,7 @@ def exprgen(prefix, matches):
 
   return finished_prefixes
 
+SUFFIX_REGEXP = re.compile(r'\*([\w\d\-\._]+)$')
 
 def get_mfp(path, recursive):
   """many,flat,prefix"""
@@ -159,6 +164,13 @@ def get_mfp(path, recursive):
   flat = not recursive
   many = recursive
   prefix = ""
+  suffix = ""
+
+  matches = SUFFIX_REGEXP.search(path)
+  if matches is not None:
+    suffix = matches.groups()[0]
+    path = path.removesuffix(suffix)
+
   if path[-2:] == "**":
     many = True
     flat = False
@@ -168,7 +180,20 @@ def get_mfp(path, recursive):
     flat = True
     prefix = os.path.basename(path[:-1])
 
-  return (many, flat, prefix)
+  return (many, flat, prefix, suffix)
+
+@main.command("mkdir")
+@click.argument("paths", nargs=-1)
+def _mkdir(paths):
+  """
+  Create paths on the local file system.
+  """
+  for path in paths:
+    path = normalize_path(path)
+    protocol = get_protocol(path)
+
+    if protocol == "file":
+      mkdir(path.replace("file://", "", 1))
 
 @main.command()
 @click.argument("source", nargs=-1)
@@ -267,7 +292,7 @@ def _cp_single(
 
   # The else clause here is to handle single file transfers
   srcpath = nsrc if issrcdir else os.path.dirname(nsrc)
-  many, flat, prefix = get_mfp(nsrc, recursive)
+  many, flat, prefix, suffix = get_mfp(nsrc, recursive)
 
   if issrcdir and not many:
     print(f"cloudfiles: {source} is a directory (not copied).")
@@ -302,6 +327,9 @@ def _cp_single(
     compression = False
 
   if not isinstance(xferpaths, str):
+    if suffix:
+      xferpaths = ( x for x in xferpaths if x.endswith(suffix) )
+
     if parallel == 1:
       _cp(
         srcpath, destpath, compression,
@@ -494,7 +522,7 @@ def _mv_single(
 
   # The else clause here is to handle single file transfers
   srcpath = nsrc if issrcdir else os.path.dirname(nsrc)
-  many, flat, prefix = get_mfp(nsrc, recursive)
+  many, flat, prefix, suffix = get_mfp(nsrc, recursive)
 
   if issrcdir and not many:
     print(f"cloudfiles: {source} is a directory (not copied).")
@@ -524,6 +552,9 @@ def _mv_single(
       return
 
   if not isinstance(xferpaths, str):
+    if suffix:
+      xferpaths = ( x for x in xferpaths if x.endswith(suffix) )
+
     if parallel == 1:
       _mv(srcpath, destpath, progress, block_size, part_bytes, no_sign_request, xferpaths)
       return 
@@ -570,6 +601,7 @@ def touch(
   ctx, sources,
   progress, no_sign_request,
 ):
+  """Create file if it doesn't exist."""
   sources = list(map(normalize_path, sources))
   sources = [ src.replace("precomputed://", "") for src in sources ]
   pbar = tqdm(total=len(sources), desc="Touch", disable=(not progress))
@@ -701,7 +733,7 @@ def rm(ctx, paths, recursive, progress, block_size):
   singles = []
   multiples = []
   for path in paths:
-    many, flat, prefix = get_mfp(path, recursive)
+    many, flat, prefix, suffix = get_mfp(path, recursive)
     if ispathdir(path) and not many:
       print(f"cloudfiles: {path}: is a directory.")
       return
@@ -742,13 +774,15 @@ def _rm_many(path, recursive, progress, parallel, block_size):
   isdir = (ispathdir(path) or CloudFiles(npath).isdir()) 
   recursive = recursive and isdir
 
-  many, flat, prefix = get_mfp(path, recursive)
+  many, flat, prefix, suffix = get_mfp(path, recursive)
 
   cfpath = npath if isdir else os.path.dirname(npath)
   xferpaths = os.path.basename(npath)
 
   if many:
     xferpaths = CloudFiles(cfpath).list(prefix=prefix, flat=flat)
+    if suffix:
+      xferpaths = ( x for x in xferpaths if x.endswith(suffix) )
 
   if parallel == 1 or not many:
     __rm(cfpath, progress, xferpaths)
@@ -825,10 +859,13 @@ def head(paths):
   for path in paths:
     npath = normalize_path(path)
     npath = re.sub(r'\*+$', '', path)
-    many, flat, prefix = get_mfp(path, False)
+    many, flat, prefix, suffix = get_mfp(path, False)
     if many:
       cf = CloudFiles(npath)
-      res = cf.head(cf.list(prefix=prefix, flat=flat))
+      lst = cf.list(prefix=prefix, flat=flat)
+      if suffix:
+        lst = ( x for x in lst if x.endswith(suffix) )
+      res = cf.head(lst)
       results.update(res)
     else:
       cf = CloudFiles(os.path.dirname(npath))

cloud_files-5.6.1/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "5554b92", "is_release": true}