cloud-files 5.1.3__tar.gz → 5.2.0__tar.gz

{cloud_files-5.1.3 → cloud_files-5.2.0}/ChangeLog

@@ -1,6 +1,12 @@
 CHANGES
 =======
 
+5.2.0
+-----
+
+* fix: support |format: style of cloudpaths (#110)
+* fix: correct crc32c representation for s3
+
 5.1.3
 -----
 

{cloud_files-5.1.3 → cloud_files-5.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.1.3
+Version: 5.2.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-5.1.3 → cloud_files-5.2.0}/automated_test.py

@@ -538,6 +538,13 @@ def test_path_extraction():
         'dataset/layer', None, None
       ))
 
+  assert (
+    paths.extract('gs://bucket/dataset/layer|zarr2:') 
+      == ExtractedPath(
+        'zarr2', 'gs', 'bucket', 
+        'dataset/layer', None, None
+      ))
+
   okgoogle('gs://bucket/dataset/layer/')
   # shoulderror('gs://bucket/dataset/layer/info')
 

{cloud_files-5.1.3 → cloud_files-5.2.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.1.3
+Version: 5.2.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud_files-5.2.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "bee40ef", "is_release": true}

{cloud_files-5.1.3 → cloud_files-5.2.0}/cloudfiles/interfaces.py

@@ -1189,10 +1189,13 @@ class S3Interface(StorageInterface):
         **attrs
       )
     else:
+      if isinstance(content, str):
+        content = content.encode('utf8')
+
       attrs['Bucket'] = self._path.bucket
       attrs['Body'] = content
       attrs['Key'] = key
-      attrs["ChecksumCRC32C"] = str(encode_crc32c_b64(content))
+      attrs["ChecksumCRC32C"] = encode_crc32c_b64(content).decode('utf8')
       self._conn.put_object(**attrs)
 
   @retry

{cloud_files-5.1.3 → cloud_files-5.2.0}/cloudfiles/lib.py

@@ -163,6 +163,8 @@ def crc32c(binary):
   Computes the crc32c of a binary string 
   and returns it as an integer.
   """
+  if isinstance(binary, str):
+    binary = binary.encode('utf8')
   return crc32clib.crc32c(binary) # an integer
 
 def md5(binary, base=64):

{cloud_files-5.1.3 → cloud_files-5.2.0}/cloudfiles/paths.py

@@ -58,11 +58,14 @@ def update_aliases_from_file():
 
 def cloudpath_error(cloudpath):
   return yellow(f"""
-    Cloud Path must conform to [FORMAT://]PROTOCOL://PATH
+    Cloud Path must conform to one of:
+      (a) [FORMAT://]PROTOCOL://PATH
+      (b) PROTOCOL://PATH|FORMAT:
     Examples: 
       precomputed://gs://test_bucket/em
       gs://test_bucket/em
       graphene://https://example.com/image/em
+      gs://text_bucket/em/|zarr2:
 
     Supported Formats: None (precomputed), {", ".join(ALLOWED_FORMATS)}
     Supported Protocols: {", ".join(ALLOWED_PROTOCOLS)}
@@ -87,6 +90,9 @@ def mkregexp():
 CLOUDPATH_REGEXP = re.compile(mkregexp())
 BUCKET_RE = re.compile(r'^(/?[~\d\w_\.\-]+(?::\d+)?)(?:/|$)') # posix /what/a/great/path  
 
+# |neuroglancer-precomputed: or |zarr2: suffixes etc
+TAIL_FORMAT_REGEXP = re.compile('\\|(?P<fmt>[\\w\\d-]+):$') 
+
 def add_alias(alias:str, host:str):
   global ALIASES
   global ALLOWED_PROTOCOLS
@@ -291,6 +297,13 @@ def pop_protocol(cloudpath):
 def extract_format_protocol(cloudpath:str, allow_defaults=True) -> tuple:
   error = UnsupportedProtocolError(cloudpath_error(cloudpath))
 
+  cloudpath = cloudpath.removesuffix(PRECOMPUTED_SUFFIX)
+
+  m = re.search(TAIL_FORMAT_REGEXP, cloudpath)
+  if m is not None:
+    cloudpath = re.sub(TAIL_FORMAT_REGEXP, "", cloudpath)
+    cloudpath = f"{m.group('fmt')}://{cloudpath}"
+
   alias, cloudpath = resolve_alias(cloudpath)
 
   m = re.match(CLOUDPATH_REGEXP, cloudpath)
@@ -299,7 +312,6 @@ def extract_format_protocol(cloudpath:str, allow_defaults=True) -> tuple:
 
   groups = m.groups()
   cloudpath = re.sub(CLOUDPATH_REGEXP, '', cloudpath, count=1)
-  cloudpath = cloudpath.removesuffix(PRECOMPUTED_SUFFIX)
 
   fmt = m.group('fmt')
   if not fmt and allow_defaults:

cloud_files-5.1.3/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "894545a", "is_release": true}