cloud-files 6.1.2__tar.gz → 6.2.0__tar.gz

{cloud_files-6.1.2 → cloud_files-6.2.0}/ChangeLog

@@ -1,6 +1,19 @@
 CHANGES
 =======
 
+6.2.0
+-----
+
+* feat: adds cloudfiles watch net print/histogram (#124)
+
+6.1.3
+-----
+
+* fix: du for single files was broken
+* fix: use service provider
+* feat: add "service\_provider" resolver
+* fix: malformed provider/bucket groupings in rm
+
 6.1.2
 -----
 

{cloud_files-6.1.2 → cloud_files-6.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 6.1.2
+Version: 6.2.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-6.1.2 → cloud_files-6.2.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 6.1.2
+Version: 6.2.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud_files-6.2.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "a0c3192", "is_release": true}

{cloud_files-6.1.2 → cloud_files-6.2.0}/cloudfiles/interfaces.py

@@ -870,6 +870,7 @@ class GoogleCloudStorageInterface(StorageInterface):
         elif flat and '/' not in blob.name.removeprefix(path):
           yield filename
 
+
   @retry
   def subtree_size(self, prefix:str = "") -> tuple[int,int]:
     layer_path = self.get_path_to_file("")        

{cloud_files-6.1.2 → cloud_files-6.2.0}/cloudfiles/paths.py

@@ -229,6 +229,18 @@ def ascloudpath(epath):
     return f"{epath.format}://" + pth
   return pth
 
+def service_provider(epath:ExtractedPath) -> str:
+  stack = [
+    epath.alias,
+    epath.protocol,
+  ]
+  while len(stack):
+    elem = stack.pop(0)
+    if elem is None:
+      continue
+    return elem
+  return None
+
 def asprotocolpath(epath):
   pth = ''
 

{cloud_files-6.1.2 → cloud_files-6.2.0}/cloudfiles_cli/cloudfiles_cli.py

@@ -12,6 +12,7 @@ import os.path
 from tqdm import tqdm
 import shutil
 import sys
+import time
 
 # Below two lines fix MacOS warning on 
 # High Sierra and above when we are using
@@ -752,8 +753,8 @@ def _rm_singles(paths, progress, parallel, block_size):
   for path in paths:
     npath = normalize_path(path)
     extracted = cloudfiles.paths.extract(npath)
-    normalized_protocol = cloudfiles.paths.asprotocolpath(extracted)
-    cfgroups[(normalized_protocol, extracted.bucket)].append(extracted.path)
+    provider = cloudfiles.paths.service_provider(extracted)
+    cfgroups[(provider, extracted.bucket)].append(extracted.path)
   
   for group, paths in cfgroups.items():
     cfpath = f"{group[0]}://{group[1]}/"
@@ -807,16 +808,16 @@ def du(paths, grand_total, summarize, human_readable, count_files):
   """Display disk usage statistics."""
   results = []
 
-  list_data = False
+  simplified_data = True
 
   for path in paths:
     npath = normalize_path(path)
     if ispathdir(path):
       cf = CloudFiles(npath)
       if summarize:
+        simplified_data = False
         results.append(cf.subtree_size())
       else:
-        list_data = True
         results.append(cf.size(cf.list()))
     else:
       cf = CloudFiles(os.path.dirname(npath))
@@ -848,7 +849,7 @@ def du(paths, grand_total, summarize, human_readable, count_files):
   summary = {}
   num_files = 0
   for path, res in zip(paths, results):
-    if list_data:
+    if simplified_data:
       summary[path] = sum(res.values())
       num_files += len(res)
     else:
@@ -1172,4 +1173,79 @@ def alias_rm(name):
   except KeyError:
     pass
 
+@main.group("watch")
+def iostatsgroup():
+  """
+  Monitor machine IO, print stats, and make histograms.
+  """
+  pass
+
+@iostatsgroup.group("net")
+def iostats_net_group():
+  """
+  Find ethernet devices and monitor them.
+  """
+  pass
+
+
+@iostats_net_group.command("print")
+@click.option("--duration", type=float, default=0, show_default=True)
+@click.option("--interval", type=float, default=1, show_default=True)
+@click.option("--one-line", is_flag=True, default=False, show_default=True)
+def net_print(duration:float, interval:float, one_line:bool):
+  """
+  Print transmissions speeds.
+  """
+  network_sampler = IOSampler(
+    buffer_sec=max(interval * 2, 1),
+    interval=interval,
+  )
+  network_sampler.start_sampling()
+
+  def fmt(elapsed, rbps, tbps):
+    return f"t={elapsed:1.1f}s | RX: {rbps/8e6:3.2f} MB/sec ({rbps/1e9:1.2f} Gbps) | TX: {tbps/8e6:3.2f} MB/sec ({tbps/1e9:1.2f} Gbps)"
+
+  ending = '\n' if not one_line else '\r'
+
+  s = time.perf_counter()
+  try:
+    while True:
+      time.sleep(interval)
+      rbps, tbps = network_sampler.current_bps()
+      elapsed = time.perf_counter() - s
+      display = fmt(elapsed, rbps, tbps)
+      print(display, flush=True, end=ending)
+
+      if elapsed > duration and duration > 0:
+        return
+  except KeyboardInterrupt:
+    pass
+
+  network_sampler.stop_sampling()
+  rbps, tbps = network_sampler.current_bps()
+  elapsed = time.perf_counter() - s
+  display = fmt(elapsed, rbps, tbps)
+  print(display, flush=True)
+
+@iostats_net_group.command("histogram")
+@click.option("--duration", type=float, default=5, show_default=True)
+@click.option("--interval", type=float, default=1, show_default=True)
+@click.option("--filename", type=str, default="", show_default=True)
+def net_histogram(duration:float, interval:float, filename:str):
+  """
+  Produce a histogram of network speeds.
+  """
+  network_sampler = IOSampler(
+    buffer_sec=max(duration * 2, 1),
+    interval=interval,
+  )
+  network_sampler.start_sampling()
+
+  time.sleep(float(duration) + 0.1)
+  network_sampler.stop_sampling()
+
+  if filename == "":
+    filename = None
+
+  network_sampler.plot_histogram(filename=filename)
 

cloud_files-6.1.2/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "2d4d337", "is_release": true}