shareddata 6.85.8__tar.gz → 6.85.10__tar.gz

{shareddata-6.85.8/src/shareddata.egg-info → shareddata-6.85.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.8
+Version: 6.85.10
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.8 → shareddata-6.85.10}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.8',
+    version='6.85.10',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.8 → shareddata-6.85.10}/src/SharedData/Table.py

@@ -22,6 +22,7 @@ import uuid
 
 from SharedData.Logger import Logger
 from SharedData.LockWaiters import TableBusy
+from SharedData.TableRewrite import BsonIntegrityError
 from SharedData.IO.AWSS3 import S3Upload, S3Download, S3ListFolder, S3GetMtime, S3DeleteFile, UpdateModTime
 from SharedData.TableIndex import TableIndex
 from SharedData.SharedNumpy import SharedNumpy
@@ -38,6 +39,11 @@ from SharedData.TableProtocol import (IDLE, WRITE, STRUCT, DIRTY, mode, epoch,
 # orphaned (append-only) has nothing to defragment, and rewriting it would only copy identical bytes.
 BSON_DEFRAG_MIN_BYTES = 64 * 1024
 BSON_DEFRAG_MIN_FRACTION = 0.0
+# The rewrite holds the table lock and copies every live byte: for a large BSON file that must buy
+# something. Above BSON_DEFRAG_LARGE_BYTES of allocated BSON, compact once at least this fraction is orphaned
+# (a 43 GB file with 23 MB orphaned is 0.05%: not worth minutes of blocked ingestion).
+BSON_DEFRAG_LARGE_BYTES = 1024 * 1024 * 1024
+BSON_DEFRAG_LARGE_MIN_FRACTION = 0.01
 
 
 class TableNotFound(FileNotFoundError):
@@ -1623,12 +1629,43 @@ class Table:
         usage['orphaned'] = max(usage['used'] - usage['live'], 0)
         return usage
 
+    def _bson_mismatches(self, pointers, sizes, live_idx):
+        """Rows whose pointer does not lead to a BSON document of exactly the recorded size: the int32 at
+        the pointer is the document length, and a document ends with a zero byte."""
+        import mmap
+        bad = []
+        if not len(live_idx):
+            return np.zeros(0, dtype=np.int64)
+        with open(self.bsonpath, 'rb') as source:
+            view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+            data = None
+            try:
+                data = np.frombuffer(view, dtype=np.uint8)
+                span = np.arange(4)
+                for first in range(0, len(live_idx), 1 << 20):
+                    rows = live_idx[first:first + (1 << 20)]
+                    short = sizes[rows] < 5                      # the smallest document, {}, is 5 bytes
+                    bad.append(rows[short])
+                    rows = rows[~short]
+                    start, length = pointers[rows], sizes[rows]
+                    head = data[start[:, None] + span].astype(np.int64)
+                    declared = head[:, 0] | (head[:, 1] << 8) | (head[:, 2] << 16) | (head[:, 3] << 24)
+                    ends_with_zero = data[start + length - 1] == 0
+                    bad.append(rows[(declared != length) | ~ends_with_zero])
+            finally:
+                data = None                                      # release the buffer export before closing the map
+                view.close()
+        return np.concatenate(bad) if bad else np.zeros(0, dtype=np.int64)
+
     def compact_bson_if_worthwhile(self):
         """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
         usage = self.bson_usage()
         if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
                 or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
             return None
+        if usage['used'] > BSON_DEFRAG_LARGE_BYTES \
+                and usage['orphaned'] < BSON_DEFRAG_LARGE_MIN_FRACTION * usage['used']:
+            return None
         return self.defragment_bson(acquire=False)
 
     def _defragment_bson_disk(self, chunk_size, acquire):
@@ -1671,6 +1708,13 @@ class Table:
             original_size = self.bsonpath.stat().st_size
             if len(live_idx) and (np.any(pointers[live_idx] < 0) or np.any(pointers[live_idx] + sizes[live_idx] > original_size)):
                 raise RewriteError(f'{self.relpath}: a record points outside its BSON file; nothing was changed')
+            mismatched = self._bson_mismatches(pointers, sizes, live_idx)
+            if len(mismatched):
+                from SharedData.TableRewrite import BsonIntegrityError
+                raise BsonIntegrityError(
+                    f'{self.relpath}: {len(mismatched)} of {len(live_idx)} records do not point at a BSON document '
+                    f'of their recorded size (first rows {mismatched[:5].tolist()}); not compacted, '
+                    'the unreferenced bytes may be the documents those records lost')
             new_offsets = np.zeros(len(live_idx), dtype=np.int64)
             if len(live_idx):
                 new_offsets[1:] = np.cumsum(sizes[live_idx])[:-1]
@@ -1720,7 +1764,8 @@ class Table:
 
             with structure(self):
                 publish_rewrite(self, None, 'defrag', start=0, transform=move_pointers,
-                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)},
+                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001),
+                                                'isidxcreated': int(self.hdr['isidxcreated'])},        # pointers only: the index stays valid
                                 bson=dict(token=token, stage=stage, sha256=digest.hexdigest(), size=new_size))
             stats = {'original_size': original_size, 'new_size': new_size,
                      'reclaimed': original_size - new_size, 'live_bytes': payload, 'live_rows': int(len(live_idx))}
@@ -2007,7 +2052,13 @@ class Table:
             if remote and self.is_schemaless and self.type == 1 and self.bsonpath.exists():
                 remote_bson_mtime = S3GetMtime(str(self.bsonpath) + '.gzip', self.database_folder)
                 if force_write or remote_bson_mtime is None or remote_bson_mtime < mtime:
-                    if self.compact_bson_if_worthwhile() is not None:
+                    try:
+                        stats = self.compact_bson_if_worthwhile()
+                    except BsonIntegrityError as error:
+                        # Loud, every night, but the upload goes on as it was: nothing is rewritten.
+                        Logger.log.error('BSON compaction skipped: %s' % error)
+                        stats = None
+                    if stats is not None:
                         compacted = True
                         # New pointers: head, tail and BSON are all rewritten and all uploaded, under a
                         # publication time later than anything the remote holds, so consumers see the change.
@@ -2967,11 +3018,18 @@ class Table:
                 self._set_mode(DIRTY)
             # A nested alias may follow a failed publication by this thread;
             # only an in-progress structural operation of this thread is STRUCT.
+            self._rewrite_kept_index = False
             if self.type == 1 and mode(self.mutex['type']) != STRUCT:
                 from SharedData.TableRewrite import recover_rewrite
                 recover_rewrite(self)
+            # A pending defrag descriptor means the dead holder was only in the defrag (one holder at a time, a
+            # crashed upsert would have been repaired before it could start): keys and rows never moved, so the
+            # index it left is valid and the dirty repair must not rebuild it (a rebuild can be refused).
+            kept_index = getattr(self, '_rewrite_kept_index', False)
             self._refresh_mapping(dirty=dirty)
-            if dirty and self.hdr is not None:
+            if kept_index and self.hasindex:
+                self.index.rememmap()           # keep the index, but map its files as they are now (the dirty refresh skips it)
+            if dirty and self.hdr is not None and not kept_index:
                 self.hdr['isidxcreated'] = 0
                 if self.hasindex:
                     self.index.initialize()

{shareddata-6.85.8 → shareddata-6.85.10}/src/SharedData/TableProtocol.py

@@ -154,9 +154,15 @@ def structure(table):
     try:
         yield
     except BaseException:
-        table._set_mode(DIRTY)
-        if table.hdr is not None:
-            table.hdr['isidxcreated'] = 0
+        if getattr(table, '_rewrite_live_untouched', False):
+            # A defrag that failed before it replaced any live file: the table is exactly as it was, so it is
+            # neither dirty nor does its index need a rebuild (which a legacy table can refuse).
+            table._rewrite_live_untouched = False
+            table._set_mode(previous)
+        else:
+            table._set_mode(DIRTY)
+            if table.hdr is not None:
+                table.hdr['isidxcreated'] = 0
         raise
     else:
         word = int(table.mutex['type'])

{shareddata-6.85.8 → shareddata-6.85.10}/src/SharedData/TableRewrite.py

@@ -18,6 +18,11 @@ class RewriteError(RuntimeError):
     pass
 
 
+class BsonIntegrityError(RewriteError):
+    """Records whose bson_ptr/bson_size do not describe a BSON document. Compacting such a table would
+    keep only the byte ranges the records point at and discard the documents they miss."""
+
+
 def checkpoint(name):
     """Test seam, deliberately not controlled by production environment."""
 
@@ -185,6 +190,8 @@ def _load(table):
                 raise ValueError('invalid BSON digest')
         elif 'bson' in descriptor:
             raise ValueError('BSON section requires version 2')
+        if 'index_valid' in descriptor and descriptor['index_valid'] not in (0, 1):
+            raise ValueError('invalid index validity')
         return descriptor
     except Exception as error:
         raise RewriteError('invalid pending record rewrite; files preserved') from error
@@ -203,10 +210,21 @@ def _finish(table, descriptor, recovery=False):
     if 'bson' in descriptor:
         table._reset_bson_mapping()         # the BSON file was replaced together with the records
     checkpoint(prefix + 'data_attached')
-    table.hdr['isidxcreated'] = 0
-    sync_header(table)
+    # A defrag moves BSON pointers only: keys, row positions and so every index file stay valid and mapped.
+    # initialize() always resets and rebuilds the index, and the rebuild refuses legacy tables with an
+    # empty-key row ("explicit cleanup required"), so a defrag neither invalidates nor rebuilds it.
+    keeps_index = descriptor.get('operation') == 'defrag'
+    table._rewrite_kept_index = keeps_index         # acquire(): a dirty owner that just finished such a recovery must not rebuild
+    if not keeps_index:
+        table.hdr['isidxcreated'] = 0
+        sync_header(table)
+    elif descriptor.get('index_valid') is not None and int(table.hdr['isidxcreated']) != descriptor['index_valid']:
+        # structure() clears the bit when a failure follows the attach: the index files never changed, so the
+        # validity they had before the defrag is restored (descriptors of older releases do not carry it).
+        table.hdr['isidxcreated'] = descriptor['index_valid']
+        sync_header(table)
     checkpoint(prefix + 'index_invalid_synced')
-    if table.hasindex:
+    if table.hasindex and not keeps_index:
         table.index.initialize()
     sync_header(table)
     checkpoint(prefix + 'final_header_synced')
@@ -312,6 +330,9 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
         raise RewriteError('disk record publication requires writer ownership')
     if pending_path(table).exists():
         raise RewriteError('unresolved record rewrite')
+    # Until the first live file is replaced a failing defrag leaves the table untouched (structure() then
+    # restores the mode instead of marking it dirty).
+    table._rewrite_live_untouched = operation == 'defrag'
     checkpoint('struct_enter')
     table.shf_data.flush()
     sync_header(table)
@@ -388,6 +409,8 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
                       transaction=token, table=str(table.relpath).replace('\\', '/'),
                       operation=operation, start=int(start), stage=stage.name,
                       old=old, new=new)
+    if operation == 'defrag':
+        descriptor['index_valid'] = int(table.hdr['isidxcreated'])
     if bson:
         bson_stage = Path(table.bsonpath).parent / ('bson.bin.stage.' + token)
         if not bson_stage.is_file() or bson_stage.stat().st_size != bson['size']:
@@ -407,6 +430,7 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     checkpoint('prepare_installed')
     fsync_directory(stage.parent)
     checkpoint('prepare_dir_synced')
+    table._rewrite_live_untouched = False       # from the first replacement on, recovery and DIRTY apply
     if bson:
         os.replace(bson_stage, table.bsonpath)
         checkpoint('bson_replaced')

{shareddata-6.85.8 → shareddata-6.85.10/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.8
+Version: 6.85.10
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata