shareddata 6.85.7__tar.gz → 6.85.8__tar.gz

{shareddata-6.85.7/src/shareddata.egg-info → shareddata-6.85.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.7
+Version: 6.85.8
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.7 → shareddata-6.85.8}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.7',
+    version='6.85.8',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/API/routes/cache.py

@@ -8,6 +8,9 @@ import datetime
 import json
 import time
 
+import numpy as np
+from bson.objectid import ObjectId
+
 from flask import Blueprint, Response, jsonify, request
 
 from SharedData.Logger import Logger
@@ -20,10 +23,17 @@ from SharedData.API.utils import (
 
 
 class _DatetimeEncoder(json.JSONEncoder):
-    """JSON encoder that serializes datetime/date objects as ISO 8601 strings."""
+    """JSON encoder for cached values: datetime/date as ISO 8601 strings, a Mongo ObjectId as its hex
+    string, NumPy scalars and arrays as plain numbers and lists."""
     def default(self, obj):
         if isinstance(obj, (datetime.datetime, datetime.date)):
             return obj.isoformat()
+        if isinstance(obj, ObjectId):
+            return str(obj)
+        if isinstance(obj, np.generic):
+            return obj.item()
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
         return super().default(obj)
 
 

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/IO/SaveTables.py

@@ -55,7 +55,7 @@ def write_table(dftable_row: Dict[str, Any]) -> str:
             dftable_row['source'],
             tablename
         )
-        tbl.write(force_write=True)
+        tbl.write(force_write=True, strict=True)
         return f"Success: {tablename}"
     except Exception as e:
         Logger.log.error(f"Error saving {dftable_row} {e}")

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/SharedData.py

@@ -253,7 +253,11 @@ class SharedData:
                 elif len(partitioningstr) == 8:
                     partitioning = 'daily'
             else:
-                if hasindex:
+                # A new indexed table of a period shorter than a day must be partitioned. Opening an existing
+                # one (no schema given) is not creating: legacy unpartitioned tables stay readable and savable.
+                creating = (names is not None or formats is not None or value is not None
+                            or is_schemaless or size is not None)
+                if hasindex and creating:
                     if period != 'D1':
                         raise Exception(f'Please specify partitioning for {period} tables')
                 

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/SharedNumpy.py

@@ -285,7 +285,7 @@ class SharedNumpy(np.ndarray):
         """
         self.table.trim()
 
-    def write(self,force_write=False):
+    def write(self,force_write=False,strict=False):
         """
         Writes the table data to storage, optionally forcing the write operation.
         
@@ -300,7 +300,7 @@ class SharedNumpy(np.ndarray):
         if (self.table.type != 1 or sys.platform == 'linux') and (
                 self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):
             self.sort_index()
-        self.table.write(force_write)
+        self.table.write(force_write, strict)
         
     def rememmap(self):
         """

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/Table.py

@@ -34,6 +34,12 @@ from SharedData.TableProtocol import (IDLE, WRITE, STRUCT, DIRTY, mode, epoch,
                                       writer_lock, take_thread_lock, structure, structural)
 
 
+# A BSON side file is compacted before every upload that has something to return: a table with nothing
+# orphaned (append-only) has nothing to defragment, and rewriting it would only copy identical bytes.
+BSON_DEFRAG_MIN_BYTES = 64 * 1024
+BSON_DEFRAG_MIN_FRACTION = 0.0
+
+
 class TableNotFound(FileNotFoundError):
     """The requested table is absent locally and remotely, with no creation schema."""
     def __init__(self, table):
@@ -1373,6 +1379,23 @@ class Table:
         return h1[:l] == h2[:l]
 
     ############### BSON MANAGEMENT ###############
+    def _reset_bson_mapping(self):
+        """Drop the BSON mapping and reopen the file by path: after a paired publication the old
+        mapping belongs to an unlinked inode and its offsets no longer match the records."""
+        if self.bson_mmap is not None:
+            try:
+                self.bson_mmap.close()
+            except (BufferError, ValueError):
+                pass
+            self.bson_mmap = None
+        if self.bson_file_handle is not None:
+            self.bson_file_handle.close()
+            self.bson_file_handle = None
+        self.curr_bson_size = 0
+        self.prev_bson_ptr = 0
+        if self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
+            self.init_bson_mmap()
+
     def init_bson_mmap(self, from_download=False):
         """Initialize or reopen memory-mapped BSON file with proper sizing.
         
@@ -1572,6 +1595,155 @@ class Table:
         data = bytes(self.bson_mmap[offset:offset + size])
         return bson.decode(data)
 
+    def _bson_columns(self, count):
+        """bson_ptr and bson_size of the committed rows as int64 (legacy tables declare them '|V8')."""
+        pointers = self.records['bson_ptr'][:count]
+        sizes = self.records['bson_size'][:count]
+        if sizes.dtype.kind == 'V' and sizes.dtype.itemsize == 8:
+            sizes = np.ascontiguousarray(sizes).view(np.int64)
+        if pointers.dtype.kind == 'V' and pointers.dtype.itemsize == 8:
+            pointers = np.ascontiguousarray(pointers).view(np.int64)
+        return np.asarray(pointers, dtype=np.int64), np.asarray(sizes, dtype=np.int64)
+
+    def bson_usage(self):
+        """What the BSON file holds: bytes referenced by live rows, the span allocated, the file size."""
+        count = int(self.hdr['count'])
+        file_size = self.bsonpath.stat().st_size if self.bsonpath.exists() else 0
+        usage = dict(file_size=file_size, used=0, live=0, orphaned=0, live_rows=0, count=count)
+        if count == 0:
+            usage['used'] = usage['orphaned'] = int(self.hdr['next_bson_ptr'])
+            return usage
+        pointers, sizes = self._bson_columns(count)
+        live = sizes > 0
+        usage['live_rows'] = int(live.sum())
+        usage['used'] = int(self.hdr['next_bson_ptr'])             # a cursor past cleared documents is orphaned space
+        if usage['live_rows']:
+            usage['live'] = int(sizes[live].sum())
+            usage['used'] = max(int((pointers[live] + sizes[live]).max()), usage['used'])
+        usage['orphaned'] = max(usage['used'] - usage['live'], 0)
+        return usage
+
+    def compact_bson_if_worthwhile(self):
+        """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
+        usage = self.bson_usage()
+        if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
+                or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
+            return None
+        return self.defragment_bson(acquire=False)
+
+    def _defragment_bson_disk(self, chunk_size, acquire):
+        """Compact bson.bin of a disk table and move every bson_ptr, as ONE crash-safe publication.
+
+        The compacted BSON and the record image with the new pointers are built beside the live files
+        (nothing live is written), then published as a pair by TableRewrite.publish_rewrite: BSON first,
+        records second, with a version 2 descriptor that recovery rolls forward. Other processes reopen
+        both files when the epoch published on leaving STRUCT reaches them (_refresh_mapping).
+        """
+        import mmap
+        import hashlib
+        from SharedData.TableRewrite import (RewriteError, copy_metadata, file_identity, fsync_directory,
+                                             pending_path, publish_rewrite, require_rewrite_support, write_all)
+        from SharedData.TableProtocol import structure
+        require_rewrite_support()
+        if not self.is_schemaless:
+            raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
+        if acquire:
+            self.acquire()
+        stage = None
+        token = None
+        try:
+            self._sync_held_mapping()
+            # The old BSON is the (old, old) recovery baseline: make its recent writes durable before any
+            # replacement exists, or a power loss could keep durable old records over lost BSON bytes.
+            if self.bson_mmap is not None:
+                self.bson_mmap.flush()
+            if self.bsonpath.exists():
+                with open(self.bsonpath, 'rb') as baseline:
+                    os.fsync(baseline.fileno())
+            count = int(self.hdr['count'])
+            pointers, sizes = self._bson_columns(count) if count else (np.zeros(0, np.int64), np.zeros(0, np.int64))
+            live_idx = np.flatnonzero(sizes > 0)
+            if not self.bsonpath.exists():
+                if len(live_idx):
+                    raise RewriteError(f'{self.relpath}: records reference a BSON file that does not exist')
+                Logger.log.warning(f'No BSON file to defragment for {self.relpath}')
+                return {'original_size': 0, 'new_size': 0, 'reclaimed': 0}
+            original_size = self.bsonpath.stat().st_size
+            if len(live_idx) and (np.any(pointers[live_idx] < 0) or np.any(pointers[live_idx] + sizes[live_idx] > original_size)):
+                raise RewriteError(f'{self.relpath}: a record points outside its BSON file; nothing was changed')
+            new_offsets = np.zeros(len(live_idx), dtype=np.int64)
+            if len(live_idx):
+                new_offsets[1:] = np.cumsum(sizes[live_idx])[:-1]
+            payload = int(sizes[live_idx].sum()) if len(live_idx) else 0
+            token = uuid.uuid4().hex
+            stage = self.bsonpath.parent / ('bson.bin.stage.' + token)
+            digest = hashlib.sha256()
+            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+            try:
+                copy_metadata(self.bsonpath, fd)
+                if len(live_idx):
+                    with open(self.bsonpath, 'rb') as source:
+                        view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+                        try:
+                            message = f'Defragmenting BSON file: {self.relpath}'
+                            for first in tqdm(range(0, len(live_idx), chunk_size), desc=message, unit='chunk'):
+                                rows = live_idx[first:first + chunk_size]
+                                buffer = bytearray(int(sizes[rows].sum()))
+                                cursor = 0
+                                for pointer, size in zip(pointers[rows].tolist(), sizes[rows].tolist()):
+                                    buffer[cursor:cursor + size] = view[pointer:pointer + size]
+                                    cursor += size
+                                digest.update(buffer)
+                                write_all(fd, buffer)
+                        finally:
+                            view.close()
+                if payload == 0:                # keep the file mappable: a physical zero-length file is refused
+                    digest.update(b'\0')
+                    write_all(fd, b'\0')
+                os.fsync(fd)
+            finally:
+                os.close(fd)
+            fsync_directory(stage.parent)
+            new_size = max(payload, 1)
+            mapping = np.full(count, -1, dtype=np.int64)
+            mapping[live_idx] = new_offsets
+
+            def move_pointers(chunk, first):
+                moved = mapping[first:first + len(chunk)]
+                selected = moved >= 0
+                if selected.any():
+                    column = chunk['bson_ptr']
+                    if column.dtype.kind == 'V' and column.dtype.itemsize == 8:
+                        column = column.view(np.int64)
+                    column[selected] = moved[selected]
+                return chunk
+
+            with structure(self):
+                publish_rewrite(self, None, 'defrag', start=0, transform=move_pointers,
+                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)},
+                                bson=dict(token=token, stage=stage, sha256=digest.hexdigest(), size=new_size))
+            stats = {'original_size': original_size, 'new_size': new_size,
+                     'reclaimed': original_size - new_size, 'live_bytes': payload, 'live_rows': int(len(live_idx))}
+            Logger.log.info(
+                f'Defragmented {self.relpath} BSON file: {original_size/1024/1024:.2f}MB -> '
+                f'{new_size/1024/1024:.2f}MB (reclaimed {stats["reclaimed"]/1024/1024:.2f}MB)')
+            return stats
+        except Exception as e:
+            Logger.log.error(f'Failed to defragment BSON file for {self.relpath}: {e}')
+            # With a pending descriptor the stages belong to recovery; without one nothing refers to them.
+            if token is not None and not pending_path(self).exists():
+                for leftover in (self.bsonpath.parent / ('bson.bin.stage.' + token),
+                                 Path(str(self.filepath) + '.stage.' + token),
+                                 Path(str(self.filepath) + '.rewrite.prepare.' + token)):
+                    try:
+                        leftover.unlink()
+                    except OSError:
+                        pass
+            raise
+        finally:
+            if acquire:
+                self.release()
+
     def defragment_bson(self, chunk_size=100000, buffer_mb=512, acquire=True):
         """
         Defragments the BSON file IN-PLACE by creating a compacted copy, then copying it back to the original file.
@@ -1603,8 +1775,7 @@ class Table:
             Exception: If the table is not schemaless or if defragmentation fails.
         """
         if self.type == 1:
-            from SharedData.TableRewrite import RewriteError
-            raise RewriteError('disk BSON maintenance requires coordinated staged publication')
+            return self._defragment_bson_disk(chunk_size, acquire)
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
         
@@ -1795,7 +1966,7 @@ class Table:
                 self.release()
 
     ############### WRITE ###############
-    def write(self, force_write=False):
+    def write(self, force_write=False, strict=False):
         """
         Writes data to a file and uploads it concurrently using separate threads.
         
@@ -1811,19 +1982,41 @@ class Table:
         7. Handles exceptions by logging an error message and re-raising the exception.
         8. Releases the lock regardless of success or failure.
         
+        A schemaless table whose BSON file is about to be uploaded is compacted first, in this thread
+        and under this lock (the compaction is a STRUCT publication, so it cannot run in the upload
+        thread), and before the header, checksums and timestamp are prepared, so that the uploaded head
+        and tail carry the moved pointers.
+
         Parameters:
             force_write (bool): If True, forces the header to be written even if not modified.
+            strict (bool): If True an upload or flush failure raises. It is always logged with the table name.
         
         Raises:
-            Exception: If the write or upload operation fails.
+            Exception: If the write fails (or, with strict, if the upload or the local flush fails).
         """
         errmsg = ''
         self.acquire()
         try:
 
             tini = time.time()
+            from SharedData.services import s3_enabled
+            remote = s3_enabled()
             # create header
             mtime = self.hdr['mtime']
+            compacted = False
+            if remote and self.is_schemaless and self.type == 1 and self.bsonpath.exists():
+                remote_bson_mtime = S3GetMtime(str(self.bsonpath) + '.gzip', self.database_folder)
+                if force_write or remote_bson_mtime is None or remote_bson_mtime < mtime:
+                    if self.compact_bson_if_worthwhile() is not None:
+                        compacted = True
+                        # New pointers: head, tail and BSON are all rewritten and all uploaded, under a
+                        # publication time later than anything the remote holds, so consumers see the change.
+                        newest = max([m for m in (remote_bson_mtime,
+                                                  S3GetMtime(str(self.headpath) + '.gzip', self.database_folder),
+                                                  S3GetMtime(str(self.tailpath) + '.gzip', self.database_folder))
+                                      if m is not None] + [float(self.hdr['mtime'])])
+                        self.hdr['mtime'] = max(float(self.hdr['mtime']), newest + 0.001)
+                        mtime = self.hdr['mtime']
             write_head = False            
             write_head = self.fill_header(force_write)
 
@@ -1833,21 +2026,32 @@ class Table:
                 if self.bson_file_handle is not None:
                     os.fsync(self.bson_file_handle.fileno())
 
-            from SharedData.services import s3_enabled
-            if not s3_enabled():
+            if not remote:
                 self.write_file()
                 return
 
+            failures = []
+
+            def guarded(label, target, *args):
+                # A thread's exception never reaches join(): keep it, name the table, let the caller decide.
+                try:
+                    target(*args)
+                except BaseException as error:
+                    failures.append('%s: %s' % (label, error))
+                    Logger.log.error('%s %s failed: %s' % (self.relpath, label, error))
+
             thread_s3 = threading.Thread(
-                target=self.upload, args=(write_head, mtime, force_write))
+                target=guarded, args=('upload', self.upload, write_head, mtime, force_write or compacted, compacted))
             thread_s3.start()
 
-            thread_flush = threading.Thread(target=self.write_file)
+            thread_flush = threading.Thread(target=guarded, args=('flush', self.write_file))
             thread_flush.start()
 
             # join threads
             thread_s3.join()
             thread_flush.join()
+            if failures and strict:
+                raise Exception('; '.join(failures))
 
             te = time.time() - tini
             datasize = self.hdr['count'] * self.hdr['itemsize'] / 1000000
@@ -2030,33 +2234,14 @@ class Table:
         return write_head
 
     ############### UPLOAD ###############
-    def upload(self, write_head, mtime, force_write=False):
+    def upload(self, write_head, mtime, force_write=False, force_tail=False):
         """
         Uploads the head, tail, and BSON files to remote storage based on modification times and flags.
         
-        For schemaless tables, defragments the BSON file FIRST to ensure that the uploaded head/tail
-        contain the correct updated pointers for the defragmented BSON file.
+        For schemaless tables the BSON file is uploaded after head and tail. Compaction is not done here:
+        it is a structural publication and runs in write(), before the header is prepared.
         """
-        # DEFRAGMENT BSON FIRST (before uploading head/tail) for schemaless tables
-        # This ensures the pointers in records are updated before we upload them
-        if self.is_schemaless and self.bsonpath.exists():
-            bson_gzip_path = str(self.bsonpath) + '.gzip'
-            remote_bson_mtime = S3GetMtime(bson_gzip_path, self.database_folder)
-            remote_bson_is_updated = False
-            if remote_bson_mtime is not None:
-                remote_bson_is_updated = remote_bson_mtime >= mtime
-            
-            # Defragment if BSON needs upload (this updates pointers in records)
-            if not remote_bson_is_updated or force_write:
-                # Pass acquire=False because we're already in write() context with lock held
-                stats = self.defragment_bson(acquire=False)
-                
-                # Mark header as modified to force head/tail upload with new pointers
-                write_head = True
-                self.hdr['mtimehead'] = mtime
-                self.hdr['mtimetail'] = mtime
-        
-        # Now upload head/tail with UPDATED pointers
+        # head/tail carry the pointers of the BSON uploaded below (compacted by write() when it was worth it)
         remote_head_mtime = S3GetMtime(str(self.headpath)+'.gzip', self.database_folder)
         remote_head_is_updated = False
         if not remote_head_mtime is None:
@@ -2070,7 +2255,7 @@ class Table:
             remote_tail_is_updated = False
             if not remote_tail_mtime is None:
                 remote_tail_is_updated = remote_tail_mtime >= self.hdr['mtimetail']
-            if not remote_tail_is_updated:
+            if force_tail or not remote_tail_is_updated:
                 self.upload_tail(mtime)
         else:
             self.delete_tail()
@@ -2741,6 +2926,8 @@ class Table:
         if (dirty or self._mapping_epoch != current
                 or self.size != int(self.hdr['recordssize'])):
             self.attach()
+            if getattr(self, 'is_schemaless', False):
+                self._reset_bson_mapping()      # the BSON file is published together with data.bin
             if self.hasindex and not dirty:
                 self.index.rememmap()
             self._mapping_epoch = current

{shareddata-6.85.7 → shareddata-6.85.8}/src/SharedData/TableRewrite.py

@@ -134,6 +134,22 @@ def image(path):
                 sha256=digest.hexdigest())
 
 
+def file_identity(path):
+    """Cheap identity of a side file (BSON): where it lives and how big it is. Renames keep the inode."""
+    info = os.stat(path)
+    return dict(dev=info.st_dev, ino=info.st_ino, size=info.st_size)
+
+
+def file_sha256(path):
+    digest = hashlib.sha256()
+    with open(path, 'rb') as stream:
+        while True:
+            chunk = stream.read(1 << 20)
+            if not chunk:
+                return digest.hexdigest()
+            digest.update(chunk)
+
+
 def canonical(value):
     return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
 
@@ -151,13 +167,24 @@ def _load(table):
         descriptor = envelope['descriptor']
         if hashlib.sha256(canonical(descriptor)).hexdigest() != envelope['sha256']:
             raise ValueError('checksum mismatch')
-        if descriptor['version'] != 1 or descriptor['magic'] != 'SharedData-record-rewrite':
+        if descriptor['version'] not in (1, 2) or descriptor['magic'] != 'SharedData-record-rewrite':
             raise ValueError('unsupported descriptor')
         if descriptor['table'] != str(table.relpath).replace('\\', '/'):
             raise ValueError('wrong table')
         token = uuid.UUID(hex=descriptor['transaction']).hex
         if descriptor['stage'] != 'data.bin.stage.' + token:
             raise ValueError('invalid stage name')
+        if descriptor['version'] == 2:      # a record image published together with its BSON side file
+            bson = descriptor['bson']
+            if bson['stage'] != 'bson.bin.stage.' + token:
+                raise ValueError('invalid BSON stage name')
+            for key in ('old', 'new'):
+                if not all(isinstance(bson[key][field], int) for field in ('dev', 'ino', 'size')):
+                    raise ValueError('invalid BSON identity')
+            if not isinstance(bson['new']['sha256'], str) or len(bson['new']['sha256']) != 64:
+                raise ValueError('invalid BSON digest')
+        elif 'bson' in descriptor:
+            raise ValueError('BSON section requires version 2')
         return descriptor
     except Exception as error:
         raise RewriteError('invalid pending record rewrite; files preserved') from error
@@ -173,6 +200,8 @@ def _finish(table, descriptor, recovery=False):
     table.hasindex = bool(header['hasindex'])
     table.attach()
     table._index_bundle = None
+    if 'bson' in descriptor:
+        table._reset_bson_mapping()         # the BSON file was replaced together with the records
     checkpoint(prefix + 'data_attached')
     table.hdr['isidxcreated'] = 0
     sync_header(table)
@@ -188,6 +217,13 @@ def _finish(table, descriptor, recovery=False):
         if image(stage) != descriptor['new']:
             raise RewriteError('pending stage identity mismatch; files preserved')
         stage.unlink()
+    if 'bson' in descriptor:
+        bson_stage = Path(table.filepath).parent / descriptor['bson']['stage']
+        if bson_stage.exists():
+            if file_identity(bson_stage)['size'] != descriptor['bson']['new']['size'] \
+                    or file_sha256(bson_stage) != descriptor['bson']['new']['sha256']:
+                raise RewriteError('pending BSON stage identity mismatch; files preserved')
+            bson_stage.unlink()
     pending_path(table).unlink()
     checkpoint(prefix + 'descriptor_unlinked')
     fsync_directory(Path(table.filepath).parent)
@@ -195,10 +231,44 @@ def _finish(table, descriptor, recovery=False):
     table._rewrite_epoch_pending = True
 
 
+def _bson_state(table, descriptor):
+    """'old' or 'new' for the installed BSON file; anything else is refused, never guessed."""
+    bson = descriptor['bson']
+    try:
+        installed = file_identity(Path(table.filepath).parent / 'bson.bin')
+    except OSError as error:
+        raise RewriteError('missing authoritative BSON file; files preserved') from error
+    if (installed['dev'], installed['ino'], installed['size']) == (bson['old']['dev'], bson['old']['ino'], bson['old']['size']):
+        return 'old'
+    if installed['size'] == bson['new']['size'] and installed['ino'] == bson['new']['ino'] \
+            and file_sha256(Path(table.filepath).parent / 'bson.bin') == bson['new']['sha256']:
+        return 'new'
+    raise RewriteError('unknown authoritative BSON file; files preserved')
+
+
+def _roll_forward_pair(table, descriptor, actual, stage):
+    """Records and BSON are published as a pair: BSON first, then the records. A crash in between leaves
+    the new BSON next to the old records; the complete, validated stages are still there, so finish it.
+    (old, old): nothing was installed, the stages are discarded by _finish. (new, new): only cleanup is left.
+    (new records, old BSON) cannot arise from this order and is refused."""
+    folder = Path(table.filepath).parent
+    records = 'old' if actual == descriptor['old'] else 'new'
+    bson = _bson_state(table, descriptor)
+    if (records, bson) == ('new', 'old'):
+        raise RewriteError('unknown record/BSON pair state; files preserved')
+    if (records, bson) == ('old', 'new'):
+        if not stage.exists() or image(stage) != descriptor['new']:
+            raise RewriteError('missing record stage for the installed BSON file; files preserved')
+        checkpoint('recovery_pair_roll_forward')
+        os.replace(stage, table.filepath)
+        fsync_directory(folder)
+        checkpoint('recovery_pair_data_replaced')
+
+
 def recover_rewrite(table):
     """Resolve old/new identity under pid lock, even after loss of all shm."""
-    if table.type != 1 or not pending_path(table).exists():
-        return False
+    if table.type != 1 or getattr(table, 'filepath', None) is None or not pending_path(table).exists():
+        return False        # no path yet: __init__ failed before naming the file (free() acquires such a table)
     require_rewrite_support()
     if not table._owns_lock():
         raise RewriteError('record recovery requires writer ownership')
@@ -214,6 +284,8 @@ def recover_rewrite(table):
         stage = Path(table.filepath).parent / descriptor['stage']
         if stage.exists() and image(stage) != descriptor['new']:
             raise RewriteError('pending stage identity mismatch; files preserved')
+        if 'bson' in descriptor:
+            _roll_forward_pair(table, descriptor, actual, stage)
         checkpoint('recovery_validated')
         table._rewrite_recovering = True
         try:
@@ -223,11 +295,17 @@ def recover_rewrite(table):
     return True
 
 
-def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_flag=None):
+def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_flag=None,
+                    transform=None, header_updates=None, bson=None):
     """Copy selected committed rows to a complete file, then atomically publish.
 
     `order` is a source-index vector, or None for the unchanged committed rows.
     No record writes target the live mmap. Exceptions retain pending state.
+
+    transform(chunk, first_row) may edit a private copy of each chunk (for example to move BSON pointers),
+    header_updates is a dict of header fields for the replacement. `bson` = dict(token, stage, sha256, size)
+    names a complete, synced BSON replacement built beside bson.bin: both files are then published as a pair
+    (BSON first, records second, a version 2 descriptor makes the pair recoverable).
     """
     require_rewrite_support()
     if table.type != 1 or not table._owns_lock():
@@ -265,7 +343,9 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     if sorted_flag is not None:
         header['isidxsorted'] = sorted_flag
     header['minchgid'] = min(int(header['minchgid']), start)
-    token = uuid.uuid4().hex
+    token = bson['token'] if bson else uuid.uuid4().hex
+    for field, value in (header_updates or {}).items():
+        header[field] = value
     stage = Path(str(table.filepath) + '.stage.' + token)
     prepare = Path(str(table.filepath) + '.rewrite.prepare.' + token)
     source_stat = os.stat(table.filepath)
@@ -287,6 +367,8 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
         for offset in range(0, count, chunk_size):
             end = min(count, offset + chunk_size)
             chunk = rows[offset:end] if order is None else rows[order[offset:end]]
+            if transform is not None:
+                chunk = transform(np.array(chunk, copy=True), offset)
             output = chunk.tobytes()
             expected_hash.update(output)
             write_all(fd, output)
@@ -302,10 +384,16 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     if (new['sha256'] != expected_hash.hexdigest() or new['count'] != count
             or new['capacity'] != capacity or image(table.filepath) != old):
         raise RewriteError('replacement validation failed; live records unchanged')
-    descriptor = dict(magic='SharedData-record-rewrite', version=1,
+    descriptor = dict(magic='SharedData-record-rewrite', version=2 if bson else 1,
                       transaction=token, table=str(table.relpath).replace('\\', '/'),
                       operation=operation, start=int(start), stage=stage.name,
                       old=old, new=new)
+    if bson:
+        bson_stage = Path(table.bsonpath).parent / ('bson.bin.stage.' + token)
+        if not bson_stage.is_file() or bson_stage.stat().st_size != bson['size']:
+            raise RewriteError('BSON stage missing or of the wrong size; live files unchanged')
+        descriptor['bson'] = dict(stage=bson_stage.name, old=file_identity(table.bsonpath),
+                                  new=dict(file_identity(bson_stage), sha256=bson['sha256']))
     envelope = dict(descriptor=descriptor, sha256=hashlib.sha256(canonical(descriptor)).hexdigest())
     fd = os.open(prepare, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
     try:
@@ -319,6 +407,11 @@ def publish_rewrite(table, order, operation, *, capacity=None, start=0, sorted_f
     checkpoint('prepare_installed')
     fsync_directory(stage.parent)
     checkpoint('prepare_dir_synced')
+    if bson:
+        os.replace(bson_stage, table.bsonpath)
+        checkpoint('bson_replaced')
+        fsync_directory(stage.parent)
+        checkpoint('bson_dir_synced')
     os.replace(stage, table.filepath)
     checkpoint('data_replaced')
     fsync_directory(stage.parent)

{shareddata-6.85.7 → shareddata-6.85.8/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.7
+Version: 6.85.8
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata