shareddata 6.85.9__tar.gz → 6.85.10__tar.gz

{shareddata-6.85.9/src/shareddata.egg-info → shareddata-6.85.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.9
+Version: 6.85.10
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.9 → shareddata-6.85.10}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.9',
+    version='6.85.10',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.9 → shareddata-6.85.10}/src/SharedData/Table.py

@@ -1764,7 +1764,8 @@ class Table:
 
             with structure(self):
                 publish_rewrite(self, None, 'defrag', start=0, transform=move_pointers,
-                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)},
+                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001),
+                                                'isidxcreated': int(self.hdr['isidxcreated'])},        # pointers only: the index stays valid
                                 bson=dict(token=token, stage=stage, sha256=digest.hexdigest(), size=new_size))
             stats = {'original_size': original_size, 'new_size': new_size,
                      'reclaimed': original_size - new_size, 'live_bytes': payload, 'live_rows': int(len(live_idx))}
@@ -3017,11 +3018,18 @@ class Table:
                 self._set_mode(DIRTY)
             # A nested alias may follow a failed publication by this thread;
             # only an in-progress structural operation of this thread is STRUCT.
+            self._rewrite_kept_index = False
             if self.type == 1 and mode(self.mutex['type']) != STRUCT:
                 from SharedData.TableRewrite import recover_rewrite
                 recover_rewrite(self)
+            # A pending defrag descriptor means the dead holder was only in the defrag (one holder at a time, a
+            # crashed upsert would have been repaired before it could start): keys and rows never moved, so the
+            # index it left is valid and the dirty repair must not rebuild it (a rebuild can be refused).
+            kept_index = getattr(self, '_rewrite_kept_index', False)
             self._refresh_mapping(dirty=dirty)
-            if dirty and self.hdr is not None:
+            if kept_index and self.hasindex:
+                self.index.rememmap()           # keep the index, but map its files as they are now (the dirty refresh skips it)
+            if dirty and self.hdr is not None and not kept_index:
                 self.hdr['isidxcreated'] = 0
                 if self.hasindex:
                     self.index.initialize()

{shareddata-6.85.9 → shareddata-6.85.10}/src/SharedData/TableProtocol.py

@@ -154,9 +154,15 @@ def structure(table):
     try:
         yield
     except BaseException:
-        table._set_mode(DIRTY)
-        if table.hdr is not None:
-            table.hdr['isidxcreated'] = 0
+        if getattr(table, '_rewrite_live_untouched', False):
+            # A defrag that failed before it replaced any live file: the table is exactly as it was, so it is
+            # neither dirty nor does its index need a rebuild (which a legacy table can refuse).
+            table._rewrite_live_untouched = False
+            table._set_mode(previous)
+        else:
+            table._set_mode(DIRTY)
+            if table.hdr is not None:
+                table.hdr['isidxcreated'] = 0
         raise
     else:
         word = int(table.mutex['type'])

{shareddata-6.85.9 → shareddata-6.85.10}/src/SharedData/TableRewrite.py

@@ -190,6 +190,8 @@ def _load(table):
                 raise ValueError('invalid BSON digest')
         elif 'bson' in descriptor:
             raise ValueError('BSON section requires version 2')
+        if 'index_valid' in descriptor and descriptor['index_valid'] not in (0, 1):
+            raise ValueError('invalid index validity')
         return descriptor
     except Exception as error:
         raise RewriteError('invalid pending record rewrite; files preserved') from error
@@ -208,10 +210,21 @@ def _finish(table, descriptor, recovery=False):
     if 'bson' in descriptor:
         table._reset_bson_mapping()         # the BSON file was replaced together with the records
     checkpoint(prefix + 'data_attached')
-    table.hdr['isidxcreated'] = 0
-    sync_header(table)
+    # A defrag moves BSON pointers only: keys, row positions and so every index file stay valid and mapped.
+    # initialize() always resets and rebuilds the index, and the rebuild refuses legacy tables with an
+    # empty-key row ("explicit cleanup required"), so a defrag neither invalidates nor rebuilds it.
+    keeps_index = descriptor.get('operation') == 'defrag'
+    table._rewrite_kept_index = keeps_index         # acquire(): a dirty owner that just finished such a recovery must not rebuild
+    if not keeps_index:
+        table.hdr['isidxcreated'] = 0
+        sync_header(table)
+    elif descriptor.get('index_valid') is not None and int(table.hdr['isidxcreated']) != descriptor['index_valid']:
+        # structure() clears the bit when a failure follows the attach: the index files never changed, so the
+        # validity they had before the defrag is restored (descriptors of older releases do not carry it).
+        table.hdr['isidxcreated'] = descriptor['index_valid']
+        sync_header(table)
     checkpoint(prefix + 'index_invalid_synced')
-    if table.hasindex:
+    if table.hasindex and not keeps_index:
         table.index.initialize()
     sync_header(table)
     checkpoint(prefix + 'final_header_synced')
@@ -317,6 +330,9 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
         raise RewriteError('disk record publication requires writer ownership')
     if pending_path(table).exists():
         raise RewriteError('unresolved record rewrite')
+    # Until the first live file is replaced a failing defrag leaves the table untouched (structure() then
+    # restores the mode instead of marking it dirty).
+    table._rewrite_live_untouched = operation == 'defrag'
     checkpoint('struct_enter')
     table.shf_data.flush()
     sync_header(table)
@@ -393,6 +409,8 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
                       transaction=token, table=str(table.relpath).replace('\\', '/'),
                       operation=operation, start=int(start), stage=stage.name,
                       old=old, new=new)
+    if operation == 'defrag':
+        descriptor['index_valid'] = int(table.hdr['isidxcreated'])
     if bson:
         bson_stage = Path(table.bsonpath).parent / ('bson.bin.stage.' + token)
         if not bson_stage.is_file() or bson_stage.stat().st_size != bson['size']:
@@ -412,6 +430,7 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     checkpoint('prepare_installed')
     fsync_directory(stage.parent)
     checkpoint('prepare_dir_synced')
+    table._rewrite_live_untouched = False       # from the first replacement on, recovery and DIRTY apply
     if bson:
         os.replace(bson_stage, table.bsonpath)
         checkpoint('bson_replaced')

{shareddata-6.85.9 → shareddata-6.85.10/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.9
+Version: 6.85.10
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata