shareddata 6.85.4__tar.gz → 6.85.6__tar.gz

{shareddata-6.85.4/src/shareddata.egg-info → shareddata-6.85.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.4
+Version: 6.85.6
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.4 → shareddata-6.85.6}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.4',
+    version='6.85.6',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.4 → shareddata-6.85.6}/src/SharedData/IO/AWSS3.py

@@ -299,6 +299,32 @@ def S3Upload(file_io, path, mtime, database_folder=None):
         Logger.log.error(Logger.user+' Uploading to S3 '+path+' ERROR!')
         raise Exception(Logger.user+' Uploading to S3 '+path+' ERROR!')
 
+_batch_delete_refused = False
+
+
+def _delete_objects(bucket, batch):
+    """
+    Delete the keys of `batch` ([{'Key': ...}, ...]) with one DeleteObjects call.
+
+    The MinIO behind this bucket demands a Content-MD5 header on DeleteObjects, and boto3 1.36 and
+    later send a CRC32 checksum instead, so the call fails with MissingContentMD5. When that happens
+    the keys are deleted one by one (DeleteObject has no body to checksum) and the batch call is not
+    tried again in this process.
+    """
+    global _batch_delete_refused
+    if not _batch_delete_refused:
+        try:
+            bucket.delete_objects(Delete={'Objects': batch})
+            return
+        except Exception as error:
+            code = getattr(error, 'response', {}).get('Error', {}).get('Code')
+            if code != 'MissingContentMD5':
+                raise
+            _batch_delete_refused = True
+    for item in batch:
+        bucket.Object(item['Key']).delete()
+
+
 def S3DeleteTable(remote_path):
     """
     Deletes the files 'head.bin.gzip' and 'tail.bin.gzip' from an S3 bucket at the specified remote path.
@@ -326,7 +352,7 @@ def S3DeleteTable(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -353,7 +379,7 @@ def S3DeleteTimeseries(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -383,7 +409,7 @@ def S3DeleteFolder(remote_path):
         batch_size = 1000  # Adjust if necessary
         for i in range(0, len(delete_us), batch_size):
             batch = delete_us[i:i+batch_size]
-            bucket.delete_objects(Delete={'Objects': batch})
+            _delete_objects(bucket, batch)
     except Exception as e:
         Logger.log.error(f"S3Delete {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3Delete {remote_path} ERROR!\n{str(e)}")
@@ -403,7 +429,7 @@ def S3DeleteFile(remote_path: str) -> None:
     """
     try:        
         s3, bucket = S3GetSession()
-        response = bucket.delete_objects(Delete={'Objects': [{'Key': remote_path}]})
+        _delete_objects(bucket, [{'Key': remote_path}])
     except Exception as e:
         Logger.log.error(f"S3DeleteFile {remote_path} ERROR!\n{str(e)}")
         raise Exception(f"S3DeleteFile {remote_path} ERROR!\n{str(e)}")    

{shareddata-6.85.4 → shareddata-6.85.6}/src/SharedData/TimeSeriesDisk.py

@@ -124,8 +124,7 @@ class TimeSeriesDisk:
 
             if value is not None:
                 # mutex() acquired the non-reentrant tag lock for initialization.
-                self._write_values_locked(value.index, value.columns, value.to_numpy(),
-                                          add_columns=True, flush=True)
+                self._write_constructor_value_locked(value)
         finally:
             self.release()
 
@@ -188,6 +187,23 @@ class TimeSeriesDisk:
         finally:
             self.release()
 
+    def _write_constructor_value_locked(self, value):
+        """
+        The `value=` of the constructor, with the meaning it always had: rows whose timestamp is not
+        a slot of the series (a weekend, a date after its end) are skipped, not an error. Routines
+        build the frame from other tables and hand it over as is. The exact-timestamp check belongs to
+        write_values(), the explicit bulk API.
+        """
+        self.ensure_columns(value.columns, _locked=True)
+        tidx = self.get_loc_timestamp(value.index.values.astype(np.int64) / 10**9)
+        skipped = int(np.isnan(tidx).sum())
+        if skipped:
+            Logger.log.warning('%s: %d of %d rows of value are not on the series time index and were skipped'
+                               % (self.relpath, skipped, len(tidx)))
+        sidx = self.columns.get_indexer(value.columns).astype(np.float64)
+        self.setValuesJit(self.shf, tidx, sidx, np.asarray(value.to_numpy(), dtype=np.float64))
+        self.shf.flush()
+
     def _write_values_locked(self, index, columns, values, *, add_columns, flush):
         index = pd.DatetimeIndex(index)
         columns = pd.Index(columns)

{shareddata-6.85.4 → shareddata-6.85.6/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.4
+Version: 6.85.6
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata