shareddata 6.85.8__tar.gz → 6.85.9__tar.gz

{shareddata-6.85.8/src/shareddata.egg-info → shareddata-6.85.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.8
+Version: 6.85.9
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.8 → shareddata-6.85.9}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.8',
+    version='6.85.9',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.8 → shareddata-6.85.9}/src/SharedData/Table.py

@@ -22,6 +22,7 @@ import uuid
 
 from SharedData.Logger import Logger
 from SharedData.LockWaiters import TableBusy
+from SharedData.TableRewrite import BsonIntegrityError
 from SharedData.IO.AWSS3 import S3Upload, S3Download, S3ListFolder, S3GetMtime, S3DeleteFile, UpdateModTime
 from SharedData.TableIndex import TableIndex
 from SharedData.SharedNumpy import SharedNumpy
@@ -38,6 +39,11 @@ from SharedData.TableProtocol import (IDLE, WRITE, STRUCT, DIRTY, mode, epoch,
 # orphaned (append-only) has nothing to defragment, and rewriting it would only copy identical bytes.
 BSON_DEFRAG_MIN_BYTES = 64 * 1024
 BSON_DEFRAG_MIN_FRACTION = 0.0
+# The rewrite holds the table lock and copies every live byte: for a large BSON file that must buy
+# something. Above BSON_DEFRAG_LARGE_BYTES of allocated BSON, compact once at least this fraction is orphaned
+# (a 43 GB file with 23 MB orphaned is 0.05%: not worth minutes of blocked ingestion).
+BSON_DEFRAG_LARGE_BYTES = 1024 * 1024 * 1024
+BSON_DEFRAG_LARGE_MIN_FRACTION = 0.01
 
 
 class TableNotFound(FileNotFoundError):
@@ -1623,12 +1629,43 @@ class Table:
         usage['orphaned'] = max(usage['used'] - usage['live'], 0)
         return usage
 
+    def _bson_mismatches(self, pointers, sizes, live_idx):
+        """Rows whose pointer does not lead to a BSON document of exactly the recorded size: the int32 at
+        the pointer is the document length, and a document ends with a zero byte."""
+        import mmap
+        bad = []
+        if not len(live_idx):
+            return np.zeros(0, dtype=np.int64)
+        with open(self.bsonpath, 'rb') as source:
+            view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+            data = None
+            try:
+                data = np.frombuffer(view, dtype=np.uint8)
+                span = np.arange(4)
+                for first in range(0, len(live_idx), 1 << 20):
+                    rows = live_idx[first:first + (1 << 20)]
+                    short = sizes[rows] < 5                      # the smallest document, {}, is 5 bytes
+                    bad.append(rows[short])
+                    rows = rows[~short]
+                    start, length = pointers[rows], sizes[rows]
+                    head = data[start[:, None] + span].astype(np.int64)
+                    declared = head[:, 0] | (head[:, 1] << 8) | (head[:, 2] << 16) | (head[:, 3] << 24)
+                    ends_with_zero = data[start + length - 1] == 0
+                    bad.append(rows[(declared != length) | ~ends_with_zero])
+            finally:
+                data = None                                      # release the buffer export before closing the map
+                view.close()
+        return np.concatenate(bad) if bad else np.zeros(0, dtype=np.int64)
+
     def compact_bson_if_worthwhile(self):
         """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
         usage = self.bson_usage()
         if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
                 or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
             return None
+        if usage['used'] > BSON_DEFRAG_LARGE_BYTES \
+                and usage['orphaned'] < BSON_DEFRAG_LARGE_MIN_FRACTION * usage['used']:
+            return None
         return self.defragment_bson(acquire=False)
 
     def _defragment_bson_disk(self, chunk_size, acquire):
@@ -1671,6 +1708,13 @@ class Table:
             original_size = self.bsonpath.stat().st_size
             if len(live_idx) and (np.any(pointers[live_idx] < 0) or np.any(pointers[live_idx] + sizes[live_idx] > original_size)):
                 raise RewriteError(f'{self.relpath}: a record points outside its BSON file; nothing was changed')
+            mismatched = self._bson_mismatches(pointers, sizes, live_idx)
+            if len(mismatched):
+                from SharedData.TableRewrite import BsonIntegrityError
+                raise BsonIntegrityError(
+                    f'{self.relpath}: {len(mismatched)} of {len(live_idx)} records do not point at a BSON document '
+                    f'of their recorded size (first rows {mismatched[:5].tolist()}); not compacted, '
+                    'the unreferenced bytes may be the documents those records lost')
             new_offsets = np.zeros(len(live_idx), dtype=np.int64)
             if len(live_idx):
                 new_offsets[1:] = np.cumsum(sizes[live_idx])[:-1]
@@ -2007,7 +2051,13 @@ class Table:
             if remote and self.is_schemaless and self.type == 1 and self.bsonpath.exists():
                 remote_bson_mtime = S3GetMtime(str(self.bsonpath) + '.gzip', self.database_folder)
                 if force_write or remote_bson_mtime is None or remote_bson_mtime < mtime:
-                    if self.compact_bson_if_worthwhile() is not None:
+                    try:
+                        stats = self.compact_bson_if_worthwhile()
+                    except BsonIntegrityError as error:
+                        # Loud, every night, but the upload goes on as it was: nothing is rewritten.
+                        Logger.log.error('BSON compaction skipped: %s' % error)
+                        stats = None
+                    if stats is not None:
                         compacted = True
                         # New pointers: head, tail and BSON are all rewritten and all uploaded, under a
                         # publication time later than anything the remote holds, so consumers see the change.

{shareddata-6.85.8 → shareddata-6.85.9}/src/SharedData/TableRewrite.py

@@ -18,6 +18,11 @@ class RewriteError(RuntimeError):
     pass
 
 
+class BsonIntegrityError(RewriteError):
+    """Records whose bson_ptr/bson_size do not describe a BSON document. Compacting such a table would
+    keep only the byte ranges the records point at and discard the documents they miss."""
+
+
 def checkpoint(name):
     """Test seam, deliberately not controlled by production environment."""
 

{shareddata-6.85.8 → shareddata-6.85.9/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.8
+Version: 6.85.9
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata