shareddata 6.85.5__tar.gz → 6.85.7__tar.gz

{shareddata-6.85.5/src/shareddata.egg-info → shareddata-6.85.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.5
+Version: 6.85.7
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.5 → shareddata-6.85.7}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.5',
+    version='6.85.7',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.5 → shareddata-6.85.7}/src/SharedData/IO/AWSS3.py

@@ -299,6 +299,32 @@ def S3Upload(file_io, path, mtime, database_folder=None):
         Logger.log.error(Logger.user+' Uploading to S3 '+path+' ERROR!')
         raise Exception(Logger.user+' Uploading to S3 '+path+' ERROR!')
 
+_batch_delete_refused = False
+
+
+def _delete_objects(bucket, batch):
+    """
+    Delete the keys of `batch` ([{'Key': ...}, ...]) with one DeleteObjects call.
+
+    The MinIO behind this bucket demands a Content-MD5 header on DeleteObjects, and boto3 1.36 and
+    later send a CRC32 checksum instead, so the call fails with MissingContentMD5. When that happens
+    the keys are deleted one by one (DeleteObject has no body to checksum) and the batch call is not
+    tried again in this process.
+    """
+    global _batch_delete_refused
+    if not _batch_delete_refused:
+        try:
+            bucket.delete_objects(Delete={'Objects': batch})
+            return
+        except Exception as error:
+            code = getattr(error, 'response', {}).get('Error', {}).get('Code')
+            if code != 'MissingContentMD5':
+                raise
+            _batch_delete_refused = True
+    for item in batch:
+        bucket.Object(item['Key']).delete()
+
+
 def S3DeleteTable(remote_path):
     """
     Deletes the files 'head.bin.gzip' and 'tail.bin.gzip' from an S3 bucket at the specified remote path.
@@ -326,7 +352,7 @@ def S3DeleteTable(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -353,7 +379,7 @@ def S3DeleteTimeseries(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -383,7 +409,7 @@ def S3DeleteFolder(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -403,7 +429,7 @@ def S3DeleteFile(remote_path: str) -> None:
     """
     try:        
         s3, bucket = S3GetSession()
-        response = bucket.delete_objects(Delete={'Objects': [{'Key': remote_path}]})
+        _delete_objects(bucket, [{'Key': remote_path}])
     except Exception as e:
         Logger.log.error(f"S3DeleteFile {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3DeleteFile {remote_path} ERROR!\n{str(e)}")    

{shareddata-6.85.5 → shareddata-6.85.7}/src/SharedData/TableRead.py

@@ -32,7 +32,9 @@ def _open_view(table, generation):
         size = int(header['recordssize'])
         info = os.fstat(stream.fileno())
         offset = table.hdrdtype.itemsize
-        if size < 1 or info.st_size < offset + size * table.recdtype.itemsize:
+        # A table created with size=0 (the real-time partitions) has no capacity until its
+        # first extend: that is an empty table, not a short image.
+        if size < 0 or info.st_size < offset + size * table.recdtype.itemsize:
             raise RewriteError('record read image is shorter than its capacity')
         records = np.memmap(stream, table.recdtype, 'r+', offset, (size,))
     # No Table back-references: replaced bundles close as soon as their last

{shareddata-6.85.5 → shareddata-6.85.7/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.5
+Version: 6.85.7
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata