shareddata 6.85.10__tar.gz → 6.85.11__tar.gz

{shareddata-6.85.10/src/shareddata.egg-info → shareddata-6.85.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.10
+Version: 6.85.11
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.10 → shareddata-6.85.11}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.10',
+    version='6.85.11',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.10 → shareddata-6.85.11}/src/SharedData/CollectionMongoDB.py

@@ -321,7 +321,7 @@ class CollectionMongoDB:
         # Insert all prepared documents into the collection        
         result = []
         if documents_to_insert:
-            result = self._collection.insert_many(documents_to_insert)
+            result = self._collection.insert_many(documents_to_insert, ordered=False)
         
         return result        
     

{shareddata-6.85.10 → shareddata-6.85.11}/src/SharedData/Table.py

@@ -687,8 +687,18 @@ class Table:
         records, so they are removed and rebuilt from the new empty header.
         """
         from SharedData.TableRewrite import (RewriteError, fsync_directory,
-                                             require_rewrite_support)
-        require_rewrite_support()
+                                             rewrite_supported)
+        if not rewrite_supported():
+            # No POSIX rename-over-a-mapped-file: remove the old files (the shared mutex is held in STRUCT
+            # mode, so this process is their only user; another process that still maps one makes the
+            # unlink fail loudly) and create the empty image in place.
+            for name in ('data.bin', 'bson.bin', 'dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
+                try:
+                    (self.filepath.parent / name).unlink()
+                except FileNotFoundError:
+                    pass
+            self.malloc(create=True)
+            return
         final = self.filepath
         old = os.stat(final)
         # The writer lock is held: any earlier stage is from a build that died.
@@ -1659,6 +1669,10 @@ class Table:
 
     def compact_bson_if_worthwhile(self):
         """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
+        if self.type == 1:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                return None         # the pair publication needs POSIX file APIs; compaction only saves space
         usage = self.bson_usage()
         if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
                 or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
@@ -1820,6 +1834,10 @@ class Table:
             Exception: If the table is not schemaless or if defragmentation fails.
         """
         if self.type == 1:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                Logger.log.warning(f'{self.relpath} BSON not compacted: the pair publication needs POSIX file APIs')
+                return {'original_size': 0, 'new_size': 0, 'reclaimed': 0}
             return self._defragment_bson_disk(chunk_size, acquire)
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')

{shareddata-6.85.10 → shareddata-6.85.11}/src/SharedData/TableDisk.py

@@ -300,7 +300,12 @@ class TableDisk(Table):
         """
         if self.hdr['recordssize'] <= self.hdr['count']:
             return self
-        from SharedData.TableRewrite import publish_rewrite
+        from SharedData.TableRewrite import publish_rewrite, rewrite_supported
+        if not rewrite_supported():
+            # Shrinking needs a second file replacing the mapped one (Linux). Trim only returns unused space,
+            # so the table is left as it is rather than failing.
+            Logger.log.warning('%s not trimmed: shrinking a disk table needs POSIX file APIs' % self.relpath)
+            return self
         return publish_rewrite(self, None, 'trim', capacity=max(2, int(self.hdr['count'])))
 
     ############### RELOAD ###############

{shareddata-6.85.10 → shareddata-6.85.11}/src/SharedData/TableRewrite.py

@@ -38,14 +38,66 @@ def fsync_directory(path):
         os.close(fd)
 
 
+def rewrite_supported():
+    """True where the staged, crash-safe rewrite can run (Linux POSIX APIs)."""
+    return sys.platform == 'linux' and all(hasattr(os, name) for name in (
+        'O_DIRECTORY', 'fchown', 'fchmod', 'posix_fallocate',
+        'listxattr', 'getxattr', 'setxattr'))
+
+
 def require_rewrite_support():
-    if sys.platform != 'linux' or not all(hasattr(os, name) for name in (
-            'O_DIRECTORY', 'fchown', 'fchmod', 'posix_fallocate',
-            'listxattr', 'getxattr', 'setxattr')):
+    if not rewrite_supported():
         raise RewriteError('crash-safe disk record rewrites require Linux POSIX APIs; '
                            'sort/trim/rewrite recovery is unsupported on this platform')
 
 
+def _publish_in_place(table, order, operation, *, capacity, start, sorted_flag, header_updates, transform, bson):
+    """Platforms without the staged rewrite (Windows): permute the rows in the mapped image itself.
+
+    The shared mutex is held in STRUCT mode by the caller, so no other process or thread reads or writes
+    the table meanwhile and nothing needs a replacement file (which Windows could not even rename over a
+    mapped file). This is the pre-6.85 behaviour and it is NOT crash-safe: a crash inside the permutation
+    leaves a table the next owner finds in STRUCT mode, and it rebuilds the index, but it cannot undo a
+    half-moved row range. Only record-preserving operations (sort, cleanup) are supported here; whatever
+    needs a second file (trim, BSON pair) is refused.
+    """
+    if bson is not None or transform is not None or operation not in ('sort', 'cleanup'):
+        raise RewriteError(f'{operation} needs the staged rewrite, which requires Linux POSIX APIs')
+    if capacity is not None and int(capacity) != int(table.hdr['recordssize']):
+        raise RewriteError(f'{operation} with a new capacity needs the staged rewrite (Linux only)')
+    if table.type != 1 or not table._owns_lock():
+        raise RewriteError('disk record publication requires writer ownership')
+    table._sync_held_mapping()
+    old_count = int(table.hdr['count'])
+    rows = table.records.view(np.ndarray)
+    if len(rows) < old_count:
+        raise RewriteError('stale rewrite source bounds')
+    if order is None:
+        count = old_count
+    else:
+        order = np.asarray(order)
+        if np.any(order < 0) or np.any(order >= old_count):
+            raise RewriteError('replacement source out of range')
+        if operation == 'sort' and not np.array_equal(np.sort(order), np.arange(old_count)):
+            raise RewriteError('record-preserving rewrite requires a complete permutation')
+        count = len(order)
+        chosen = rows[order]                         # fancy indexing copies: safe to overwrite the source rows
+        rows[:count] = chosen
+        if count < old_count:
+            rows[count:old_count] = np.zeros((), dtype=rows.dtype)
+    table.hdr['count'] = count
+    table.hdr['isidxcreated'] = 0
+    if sorted_flag is not None:
+        table.hdr['isidxsorted'] = sorted_flag
+    table.hdr['minchgid'] = min(int(table.hdr['minchgid']), int(start))
+    for field, value in (header_updates or {}).items():
+        table.hdr[field] = value
+    table.shf_data.flush()
+    sync_header(table)
+    table.index.rebuild_index_only()
+    return table.records
+
+
 def sync_file(path):
     with open(path, 'r+b' if os.name == 'nt' else 'rb') as stream:
         os.fsync(stream.fileno())
@@ -325,7 +377,9 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     names a complete, synced BSON replacement built beside bson.bin: both files are then published as a pair
     (BSON first, records second, a version 2 descriptor makes the pair recoverable).
     """
-    require_rewrite_support()
+    if not rewrite_supported():
+        return _publish_in_place(table, order, operation, capacity=capacity, start=start, sorted_flag=sorted_flag,
+                                 header_updates=header_updates, transform=transform, bson=bson)
     if table.type != 1 or not table._owns_lock():
         raise RewriteError('disk record publication requires writer ownership')
     if pending_path(table).exists():

{shareddata-6.85.10 → shareddata-6.85.11/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.10
+Version: 6.85.11
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata