shareddata 6.85.9__tar.gz → 6.85.11__tar.gz

{shareddata-6.85.9/src/shareddata.egg-info → shareddata-6.85.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.9
+Version: 6.85.11
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.9 → shareddata-6.85.11}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.9',
+    version='6.85.11',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.9 → shareddata-6.85.11}/src/SharedData/CollectionMongoDB.py

@@ -321,7 +321,7 @@ class CollectionMongoDB:
         # Insert all prepared documents into the collection        
         result = []
         if documents_to_insert:
-            result = self._collection.insert_many(documents_to_insert)
+            result = self._collection.insert_many(documents_to_insert, ordered=False)
         
         return result        
     

{shareddata-6.85.9 → shareddata-6.85.11}/src/SharedData/Table.py

@@ -687,8 +687,18 @@ class Table:
         records, so they are removed and rebuilt from the new empty header.
         """
         from SharedData.TableRewrite import (RewriteError, fsync_directory,
-                                             require_rewrite_support)
-        require_rewrite_support()
+                                             rewrite_supported)
+        if not rewrite_supported():
+            # No POSIX rename-over-a-mapped-file: remove the old files (the shared mutex is held in STRUCT
+            # mode, so this process is their only user; another process that still maps one makes the
+            # unlink fail loudly) and create the empty image in place.
+            for name in ('data.bin', 'bson.bin', 'dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
+                try:
+                    (self.filepath.parent / name).unlink()
+                except FileNotFoundError:
+                    pass
+            self.malloc(create=True)
+            return
         final = self.filepath
         old = os.stat(final)
         # The writer lock is held: any earlier stage is from a build that died.
@@ -1659,6 +1669,10 @@ class Table:
 
     def compact_bson_if_worthwhile(self):
         """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
+        if self.type == 1:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                return None         # the pair publication needs POSIX file APIs; compaction only saves space
         usage = self.bson_usage()
         if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
                 or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
@@ -1764,7 +1778,8 @@ class Table:
 
             with structure(self):
                 publish_rewrite(self, None, 'defrag', start=0, transform=move_pointers,
-                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)},
+                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001),
+                                                'isidxcreated': int(self.hdr['isidxcreated'])},        # pointers only: the index stays valid
                                 bson=dict(token=token, stage=stage, sha256=digest.hexdigest(), size=new_size))
             stats = {'original_size': original_size, 'new_size': new_size,
                      'reclaimed': original_size - new_size, 'live_bytes': payload, 'live_rows': int(len(live_idx))}
@@ -1819,6 +1834,10 @@ class Table:
             Exception: If the table is not schemaless or if defragmentation fails.
         """
         if self.type == 1:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                Logger.log.warning(f'{self.relpath} BSON not compacted: the pair publication needs POSIX file APIs')
+                return {'original_size': 0, 'new_size': 0, 'reclaimed': 0}
             return self._defragment_bson_disk(chunk_size, acquire)
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
@@ -3017,11 +3036,18 @@ class Table:
                 self._set_mode(DIRTY)
             # A nested alias may follow a failed publication by this thread;
             # only an in-progress structural operation of this thread is STRUCT.
+            self._rewrite_kept_index = False
             if self.type == 1 and mode(self.mutex['type']) != STRUCT:
                 from SharedData.TableRewrite import recover_rewrite
                 recover_rewrite(self)
+            # A pending defrag descriptor means the dead holder was only in the defrag (one holder at a time, a
+            # crashed upsert would have been repaired before it could start): keys and rows never moved, so the
+            # index it left is valid and the dirty repair must not rebuild it (a rebuild can be refused).
+            kept_index = getattr(self, '_rewrite_kept_index', False)
             self._refresh_mapping(dirty=dirty)
-            if dirty and self.hdr is not None:
+            if kept_index and self.hasindex:
+                self.index.rememmap()           # keep the index, but map its files as they are now (the dirty refresh skips it)
+            if dirty and self.hdr is not None and not kept_index:
                 self.hdr['isidxcreated'] = 0
                 if self.hasindex:
                     self.index.initialize()

{shareddata-6.85.9 → shareddata-6.85.11}/src/SharedData/TableDisk.py

@@ -300,7 +300,12 @@ class TableDisk(Table):
         """
         if self.hdr['recordssize'] <= self.hdr['count']:
             return self
-        from SharedData.TableRewrite import publish_rewrite
+        from SharedData.TableRewrite import publish_rewrite, rewrite_supported
+        if not rewrite_supported():
+            # Shrinking needs a second file replacing the mapped one (Linux). Trim only returns unused space,
+            # so the table is left as it is rather than failing.
+            Logger.log.warning('%s not trimmed: shrinking a disk table needs POSIX file APIs' % self.relpath)
+            return self
         return publish_rewrite(self, None, 'trim', capacity=max(2, int(self.hdr['count'])))
 
     ############### RELOAD ###############

{shareddata-6.85.9 → shareddata-6.85.11}/src/SharedData/TableProtocol.py

@@ -154,9 +154,15 @@ def structure(table):
     try:
         yield
     except BaseException:
-        table._set_mode(DIRTY)
-        if table.hdr is not None:
-            table.hdr['isidxcreated'] = 0
+        if getattr(table, '_rewrite_live_untouched', False):
+            # A defrag that failed before it replaced any live file: the table is exactly as it was, so it is
+            # neither dirty nor does its index need a rebuild (which a legacy table can refuse).
+            table._rewrite_live_untouched = False
+            table._set_mode(previous)
+        else:
+            table._set_mode(DIRTY)
+            if table.hdr is not None:
+                table.hdr['isidxcreated'] = 0
         raise
     else:
         word = int(table.mutex['type'])

{shareddata-6.85.9 → shareddata-6.85.11}/src/SharedData/TableRewrite.py

@@ -38,14 +38,66 @@ def fsync_directory(path):
         os.close(fd)
 
 
+def rewrite_supported():
+    """True where the staged, crash-safe rewrite can run (Linux POSIX APIs)."""
+    return sys.platform == 'linux' and all(hasattr(os, name) for name in (
+        'O_DIRECTORY', 'fchown', 'fchmod', 'posix_fallocate',
+        'listxattr', 'getxattr', 'setxattr'))
+
+
 def require_rewrite_support():
-    if sys.platform != 'linux' or not all(hasattr(os, name) for name in (
-            'O_DIRECTORY', 'fchown', 'fchmod', 'posix_fallocate',
-            'listxattr', 'getxattr', 'setxattr')):
+    if not rewrite_supported():
         raise RewriteError('crash-safe disk record rewrites require Linux POSIX APIs; '
                            'sort/trim/rewrite recovery is unsupported on this platform')
 
 
+def _publish_in_place(table, order, operation, *, capacity, start, sorted_flag, header_updates, transform, bson):
+    """Platforms without the staged rewrite (Windows): permute the rows in the mapped image itself.
+
+    The shared mutex is held in STRUCT mode by the caller, so no other process or thread reads or writes
+    the table meanwhile and nothing needs a replacement file (which Windows could not even rename over a
+    mapped file). This is the pre-6.85 behaviour and it is NOT crash-safe: a crash inside the permutation
+    leaves a table the next owner finds in STRUCT mode, and it rebuilds the index, but it cannot undo a
+    half-moved row range. Only record-preserving operations (sort, cleanup) are supported here; whatever
+    needs a second file (trim, BSON pair) is refused.
+    """
+    if bson is not None or transform is not None or operation not in ('sort', 'cleanup'):
+        raise RewriteError(f'{operation} needs the staged rewrite, which requires Linux POSIX APIs')
+    if capacity is not None and int(capacity) != int(table.hdr['recordssize']):
+        raise RewriteError(f'{operation} with a new capacity needs the staged rewrite (Linux only)')
+    if table.type != 1 or not table._owns_lock():
+        raise RewriteError('disk record publication requires writer ownership')
+    table._sync_held_mapping()
+    old_count = int(table.hdr['count'])
+    rows = table.records.view(np.ndarray)
+    if len(rows) < old_count:
+        raise RewriteError('stale rewrite source bounds')
+    if order is None:
+        count = old_count
+    else:
+        order = np.asarray(order)
+        if np.any(order < 0) or np.any(order >= old_count):
+            raise RewriteError('replacement source out of range')
+        if operation == 'sort' and not np.array_equal(np.sort(order), np.arange(old_count)):
+            raise RewriteError('record-preserving rewrite requires a complete permutation')
+        count = len(order)
+        chosen = rows[order]                         # fancy indexing copies: safe to overwrite the source rows
+        rows[:count] = chosen
+        if count < old_count:
+            rows[count:old_count] = np.zeros((), dtype=rows.dtype)
+    table.hdr['count'] = count
+    table.hdr['isidxcreated'] = 0
+    if sorted_flag is not None:
+        table.hdr['isidxsorted'] = sorted_flag
+    table.hdr['minchgid'] = min(int(table.hdr['minchgid']), int(start))
+    for field, value in (header_updates or {}).items():
+        table.hdr[field] = value
+    table.shf_data.flush()
+    sync_header(table)
+    table.index.rebuild_index_only()
+    return table.records
+
+
 def sync_file(path):
     with open(path, 'r+b' if os.name == 'nt' else 'rb') as stream:
         os.fsync(stream.fileno())
@@ -190,6 +242,8 @@ def _load(table):
                 raise ValueError('invalid BSON digest')
         elif 'bson' in descriptor:
             raise ValueError('BSON section requires version 2')
+        if 'index_valid' in descriptor and descriptor['index_valid'] not in (0, 1):
+            raise ValueError('invalid index validity')
         return descriptor
     except Exception as error:
         raise RewriteError('invalid pending record rewrite; files preserved') from error
@@ -208,10 +262,21 @@ def _finish(table, descriptor, recovery=False):
     if 'bson' in descriptor:
         table._reset_bson_mapping()         # the BSON file was replaced together with the records
     checkpoint(prefix + 'data_attached')
-    table.hdr['isidxcreated'] = 0
-    sync_header(table)
+    # A defrag moves BSON pointers only: keys, row positions and so every index file stay valid and mapped.
+    # initialize() always resets and rebuilds the index, and the rebuild refuses legacy tables with an
+    # empty-key row ("explicit cleanup required"), so a defrag neither invalidates nor rebuilds it.
+    keeps_index = descriptor.get('operation') == 'defrag'
+    table._rewrite_kept_index = keeps_index         # acquire(): a dirty owner that just finished such a recovery must not rebuild
+    if not keeps_index:
+        table.hdr['isidxcreated'] = 0
+        sync_header(table)
+    elif descriptor.get('index_valid') is not None and int(table.hdr['isidxcreated']) != descriptor['index_valid']:
+        # structure() clears the bit when a failure follows the attach: the index files never changed, so the
+        # validity they had before the defrag is restored (descriptors of older releases do not carry it).
+        table.hdr['isidxcreated'] = descriptor['index_valid']
+        sync_header(table)
     checkpoint(prefix + 'index_invalid_synced')
-    if table.hasindex:
+    if table.hasindex and not keeps_index:
         table.index.initialize()
     sync_header(table)
     checkpoint(prefix + 'final_header_synced')
@@ -312,11 +377,16 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     names a complete, synced BSON replacement built beside bson.bin: both files are then published as a pair
     (BSON first, records second, a version 2 descriptor makes the pair recoverable).
     """
-    require_rewrite_support()
+    if not rewrite_supported():
+        return _publish_in_place(table, order, operation, capacity=capacity, start=start, sorted_flag=sorted_flag,
+                                 header_updates=header_updates, transform=transform, bson=bson)
     if table.type != 1 or not table._owns_lock():
         raise RewriteError('disk record publication requires writer ownership')
     if pending_path(table).exists():
         raise RewriteError('unresolved record rewrite')
+    # Until the first live file is replaced a failing defrag leaves the table untouched (structure() then
+    # restores the mode instead of marking it dirty).
+    table._rewrite_live_untouched = operation == 'defrag'
     checkpoint('struct_enter')
     table.shf_data.flush()
     sync_header(table)
@@ -393,6 +463,8 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
                       transaction=token, table=str(table.relpath).replace('\\', '/'),
                       operation=operation, start=int(start), stage=stage.name,
                       old=old, new=new)
+    if operation == 'defrag':
+        descriptor['index_valid'] = int(table.hdr['isidxcreated'])
     if bson:
         bson_stage = Path(table.bsonpath).parent / ('bson.bin.stage.' + token)
         if not bson_stage.is_file() or bson_stage.stat().st_size != bson['size']:
@@ -412,6 +484,7 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     checkpoint('prepare_installed')
     fsync_directory(stage.parent)
     checkpoint('prepare_dir_synced')
+    table._rewrite_live_untouched = False       # from the first replacement on, recovery and DIRTY apply
     if bson:
         os.replace(bson_stage, table.bsonpath)
         checkpoint('bson_replaced')

{shareddata-6.85.9 → shareddata-6.85.11/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.9
+Version: 6.85.11
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata