shareddata 6.85.7__tar.gz → 6.85.9__tar.gz

{shareddata-6.85.7/src/shareddata.egg-info → shareddata-6.85.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.7
+Version: 6.85.9
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.7 → shareddata-6.85.9}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.7',
+    version='6.85.9',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.7 → shareddata-6.85.9}/src/SharedData/API/routes/cache.py

@@ -8,6 +8,9 @@ import datetime
 import json
 import time
 
+import numpy as np
+from bson.objectid import ObjectId
+
 from flask import Blueprint, Response, jsonify, request
 
 from SharedData.Logger import Logger
@@ -20,10 +23,17 @@ from SharedData.API.utils import (
 
 
 class _DatetimeEncoder(json.JSONEncoder):
-    """JSON encoder that serializes datetime/date objects as ISO 8601 strings."""
+    """JSON encoder for cached values: datetime/date as ISO 8601 strings, a Mongo ObjectId as its hex
+    string, NumPy scalars and arrays as plain numbers and lists."""
     def default(self, obj):
         if isinstance(obj, (datetime.datetime, datetime.date)):
             return obj.isoformat()
+        if isinstance(obj, ObjectId):
+            return str(obj)
+        if isinstance(obj, np.generic):
+            return obj.item()
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
         return super().default(obj)
 
 

{shareddata-6.85.7 → shareddata-6.85.9}/src/SharedData/IO/SaveTables.py

@@ -55,7 +55,7 @@ def write_table(dftable_row: Dict[str, Any]) -> str:
             dftable_row['source'],
             tablename
         )
-        tbl.write(force_write=True)
+        tbl.write(force_write=True, strict=True)
         return f"Success: {tablename}"
     except Exception as e:
         Logger.log.error(f"Error saving {dftable_row} {e}")

{shareddata-6.85.7 → shareddata-6.85.9}/src/SharedData/SharedData.py

@@ -253,7 +253,11 @@ class SharedData:
                 elif len(partitioningstr) == 8:
                     partitioning = 'daily'
             else:
-                if hasindex:
+                # A new indexed table of a period shorter than a day must be partitioned. Opening an existing
+                # one (no schema given) is not creating: legacy unpartitioned tables stay readable and savable.
+                creating = (names is not None or formats is not None or value is not None
+                            or is_schemaless or size is not None)
+                if hasindex and creating:
                     if period != 'D1':
                         raise Exception(f'Please specify partitioning for {period} tables')
                 

{shareddata-6.85.7 → shareddata-6.85.9}/src/SharedData/SharedNumpy.py

@@ -285,7 +285,7 @@ class SharedNumpy(np.ndarray):
         """
         self.table.trim()
 
-    def write(self,force_write=False):
+    def write(self,force_write=False,strict=False):
         """
         Writes the table data to storage, optionally forcing the write operation.
         
@@ -300,7 +300,7 @@ class SharedNumpy(np.ndarray):
         if (self.table.type != 1 or sys.platform == 'linux') and (
                 self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):
             self.sort_index()
-        self.table.write(force_write)
+        self.table.write(force_write, strict)
         
     def rememmap(self):
         """

{shareddata-6.85.7 → shareddata-6.85.9}/src/SharedData/Table.py

@@ -22,6 +22,7 @@ import uuid
 
 from SharedData.Logger import Logger
 from SharedData.LockWaiters import TableBusy
+from SharedData.TableRewrite import BsonIntegrityError
 from SharedData.IO.AWSS3 import S3Upload, S3Download, S3ListFolder, S3GetMtime, S3DeleteFile, UpdateModTime
 from SharedData.TableIndex import TableIndex
 from SharedData.SharedNumpy import SharedNumpy
@@ -34,6 +35,17 @@ from SharedData.TableProtocol import (IDLE, WRITE, STRUCT, DIRTY, mode, epoch,
                                       writer_lock, take_thread_lock, structure, structural)
 
 
+# A BSON side file is compacted before every upload that has something to return: a table with nothing
+# orphaned (append-only) has nothing to defragment, and rewriting it would only copy identical bytes.
+BSON_DEFRAG_MIN_BYTES = 64 * 1024
+BSON_DEFRAG_MIN_FRACTION = 0.0
+# The rewrite holds the table lock and copies every live byte: for a large BSON file that must buy
+# something. Above BSON_DEFRAG_LARGE_BYTES of allocated BSON, compact once at least this fraction is orphaned
+# (a 43 GB file with 23 MB orphaned is 0.05%: not worth minutes of blocked ingestion).
+BSON_DEFRAG_LARGE_BYTES = 1024 * 1024 * 1024
+BSON_DEFRAG_LARGE_MIN_FRACTION = 0.01
+
+
 class TableNotFound(FileNotFoundError):
     """The requested table is absent locally and remotely, with no creation schema."""
     def __init__(self, table):
@@ -1373,6 +1385,23 @@ class Table:
         return h1[:l] == h2[:l]
 
     ############### BSON MANAGEMENT ###############
+    def _reset_bson_mapping(self):
+        """Drop the BSON mapping and reopen the file by path: after a paired publication the old
+        mapping belongs to an unlinked inode and its offsets no longer match the records."""
+        if self.bson_mmap is not None:
+            try:
+                self.bson_mmap.close()
+            except (BufferError, ValueError):
+                pass
+            self.bson_mmap = None
+        if self.bson_file_handle is not None:
+            self.bson_file_handle.close()
+            self.bson_file_handle = None
+        self.curr_bson_size = 0
+        self.prev_bson_ptr = 0
+        if self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
+            self.init_bson_mmap()
+
     def init_bson_mmap(self, from_download=False):
         """Initialize or reopen memory-mapped BSON file with proper sizing.
         
@@ -1572,6 +1601,193 @@ class Table:
         data = bytes(self.bson_mmap[offset:offset + size])
         return bson.decode(data)
 
+    def _bson_columns(self, count):
+        """bson_ptr and bson_size of the committed rows as int64 (legacy tables declare them '|V8')."""
+        pointers = self.records['bson_ptr'][:count]
+        sizes = self.records['bson_size'][:count]
+        if sizes.dtype.kind == 'V' and sizes.dtype.itemsize == 8:
+            sizes = np.ascontiguousarray(sizes).view(np.int64)
+        if pointers.dtype.kind == 'V' and pointers.dtype.itemsize == 8:
+            pointers = np.ascontiguousarray(pointers).view(np.int64)
+        return np.asarray(pointers, dtype=np.int64), np.asarray(sizes, dtype=np.int64)
+
+    def bson_usage(self):
+        """What the BSON file holds: bytes referenced by live rows, the span allocated, the file size."""
+        count = int(self.hdr['count'])
+        file_size = self.bsonpath.stat().st_size if self.bsonpath.exists() else 0
+        usage = dict(file_size=file_size, used=0, live=0, orphaned=0, live_rows=0, count=count)
+        if count == 0:
+            usage['used'] = usage['orphaned'] = int(self.hdr['next_bson_ptr'])
+            return usage
+        pointers, sizes = self._bson_columns(count)
+        live = sizes > 0
+        usage['live_rows'] = int(live.sum())
+        usage['used'] = int(self.hdr['next_bson_ptr'])             # a cursor past cleared documents is orphaned space
+        if usage['live_rows']:
+            usage['live'] = int(sizes[live].sum())
+            usage['used'] = max(int((pointers[live] + sizes[live]).max()), usage['used'])
+        usage['orphaned'] = max(usage['used'] - usage['live'], 0)
+        return usage
+
+    def _bson_mismatches(self, pointers, sizes, live_idx):
+        """Rows whose pointer does not lead to a BSON document of exactly the recorded size: the int32 at
+        the pointer is the document length, and a document ends with a zero byte."""
+        import mmap
+        bad = []
+        if not len(live_idx):
+            return np.zeros(0, dtype=np.int64)
+        with open(self.bsonpath, 'rb') as source:
+            view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+            data = None
+            try:
+                data = np.frombuffer(view, dtype=np.uint8)
+                span = np.arange(4)
+                for first in range(0, len(live_idx), 1 << 20):
+                    rows = live_idx[first:first + (1 << 20)]
+                    short = sizes[rows] < 5                      # the smallest document, {}, is 5 bytes
+                    bad.append(rows[short])
+                    rows = rows[~short]
+                    start, length = pointers[rows], sizes[rows]
+                    head = data[start[:, None] + span].astype(np.int64)
+                    declared = head[:, 0] | (head[:, 1] << 8) | (head[:, 2] << 16) | (head[:, 3] << 24)
+                    ends_with_zero = data[start + length - 1] == 0
+                    bad.append(rows[(declared != length) | ~ends_with_zero])
+            finally:
+                data = None                                      # release the buffer export before closing the map
+                view.close()
+        return np.concatenate(bad) if bad else np.zeros(0, dtype=np.int64)
+
+    def compact_bson_if_worthwhile(self):
+        """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
+        usage = self.bson_usage()
+        if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
+                or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
+            return None
+        if usage['used'] > BSON_DEFRAG_LARGE_BYTES \
+                and usage['orphaned'] < BSON_DEFRAG_LARGE_MIN_FRACTION * usage['used']:
+            return None
+        return self.defragment_bson(acquire=False)
+
+    def _defragment_bson_disk(self, chunk_size, acquire):
+        """Compact bson.bin of a disk table and move every bson_ptr, as ONE crash-safe publication.
+
+        The compacted BSON and the record image with the new pointers are built beside the live files
+        (nothing live is written), then published as a pair by TableRewrite.publish_rewrite: BSON first,
+        records second, with a version 2 descriptor that recovery rolls forward. Other processes reopen
+        both files when the epoch published on leaving STRUCT reaches them (_refresh_mapping).
+        """
+        import mmap
+        import hashlib
+        from SharedData.TableRewrite import (RewriteError, copy_metadata, file_identity, fsync_directory,
+                                             pending_path, publish_rewrite, require_rewrite_support, write_all)
+        from SharedData.TableProtocol import structure
+        require_rewrite_support()
+        if not self.is_schemaless:
+            raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
+        if acquire:
+            self.acquire()
+        stage = None
+        token = None
+        try:
+            self._sync_held_mapping()
+            # The old BSON is the (old, old) recovery baseline: make its recent writes durable before any
+            # replacement exists, or a power loss could keep durable old records over lost BSON bytes.
+            if self.bson_mmap is not None:
+                self.bson_mmap.flush()
+            if self.bsonpath.exists():
+                with open(self.bsonpath, 'rb') as baseline:
+                    os.fsync(baseline.fileno())
+            count = int(self.hdr['count'])
+            pointers, sizes = self._bson_columns(count) if count else (np.zeros(0, np.int64), np.zeros(0, np.int64))
+            live_idx = np.flatnonzero(sizes > 0)
+            if not self.bsonpath.exists():
+                if len(live_idx):
+                    raise RewriteError(f'{self.relpath}: records reference a BSON file that does not exist')
+                Logger.log.warning(f'No BSON file to defragment for {self.relpath}')
+                return {'original_size': 0, 'new_size': 0, 'reclaimed': 0}
+            original_size = self.bsonpath.stat().st_size
+            if len(live_idx) and (np.any(pointers[live_idx] < 0) or np.any(pointers[live_idx] + sizes[live_idx] > original_size)):
+                raise RewriteError(f'{self.relpath}: a record points outside its BSON file; nothing was changed')
+            mismatched = self._bson_mismatches(pointers, sizes, live_idx)
+            if len(mismatched):
+                from SharedData.TableRewrite import BsonIntegrityError
+                raise BsonIntegrityError(
+                    f'{self.relpath}: {len(mismatched)} of {len(live_idx)} records do not point at a BSON document '
+                    f'of their recorded size (first rows {mismatched[:5].tolist()}); not compacted, '
+                    'the unreferenced bytes may be the documents those records lost')
+            new_offsets = np.zeros(len(live_idx), dtype=np.int64)
+            if len(live_idx):
+                new_offsets[1:] = np.cumsum(sizes[live_idx])[:-1]
+            payload = int(sizes[live_idx].sum()) if len(live_idx) else 0
+            token = uuid.uuid4().hex
+            stage = self.bsonpath.parent / ('bson.bin.stage.' + token)
+            digest = hashlib.sha256()
+            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+            try:
+                copy_metadata(self.bsonpath, fd)
+                if len(live_idx):
+                    with open(self.bsonpath, 'rb') as source:
+                        view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+                        try:
+                            message = f'Defragmenting BSON file: {self.relpath}'
+                            for first in tqdm(range(0, len(live_idx), chunk_size), desc=message, unit='chunk'):
+                                rows = live_idx[first:first + chunk_size]
+                                buffer = bytearray(int(sizes[rows].sum()))
+                                cursor = 0
+                                for pointer, size in zip(pointers[rows].tolist(), sizes[rows].tolist()):
+                                    buffer[cursor:cursor + size] = view[pointer:pointer + size]
+                                    cursor += size
+                                digest.update(buffer)
+                                write_all(fd, buffer)
+                        finally:
+                            view.close()
+                if payload == 0:                # keep the file mappable: a physical zero-length file is refused
+                    digest.update(b'\0')
+                    write_all(fd, b'\0')
+                os.fsync(fd)
+            finally:
+                os.close(fd)
+            fsync_directory(stage.parent)
+            new_size = max(payload, 1)
+            mapping = np.full(count, -1, dtype=np.int64)
+            mapping[live_idx] = new_offsets
+
+            def move_pointers(chunk, first):
+                moved = mapping[first:first + len(chunk)]
+                selected = moved >= 0
+                if selected.any():
+                    column = chunk['bson_ptr']
+                    if column.dtype.kind == 'V' and column.dtype.itemsize == 8:
+                        column = column.view(np.int64)
+                    column[selected] = moved[selected]
+                return chunk
+
+            with structure(self):
+                publish_rewrite(self, None, 'defrag', start=0, transform=move_pointers,
+                                header_updates={'next_bson_ptr': payload, 'mtime': max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)},
+                                bson=dict(token=token, stage=stage, sha256=digest.hexdigest(), size=new_size))
+            stats = {'original_size': original_size, 'new_size': new_size,
+                     'reclaimed': original_size - new_size, 'live_bytes': payload, 'live_rows': int(len(live_idx))}
+            Logger.log.info(
+                f'Defragmented {self.relpath} BSON file: {original_size/1024/1024:.2f}MB -> '
+                f'{new_size/1024/1024:.2f}MB (reclaimed {stats["reclaimed"]/1024/1024:.2f}MB)')
+            return stats
+        except Exception as e:
+            Logger.log.error(f'Failed to defragment BSON file for {self.relpath}: {e}')
+            # With a pending descriptor the stages belong to recovery; without one nothing refers to them.
+            if token is not None and not pending_path(self).exists():
+                for leftover in (self.bsonpath.parent / ('bson.bin.stage.' + token),
+                                 Path(str(self.filepath) + '.stage.' + token),
+                                 Path(str(self.filepath) + '.rewrite.prepare.' + token)):
+                    try:
+                        leftover.unlink()
+                    except OSError:
+                        pass
+            raise
+        finally:
+            if acquire:
+                self.release()
+
     def defragment_bson(self, chunk_size=100000, buffer_mb=512, acquire=True):
         """
         Defragments the BSON file IN-PLACE by creating a compacted copy, then copying it back to the original file.
@@ -1603,8 +1819,7 @@ class Table:
             Exception: If the table is not schemaless or if defragmentation fails.
         """
         if self.type == 1:
-            from SharedData.TableRewrite import RewriteError
-            raise RewriteError('disk BSON maintenance requires coordinated staged publication')
+            return self._defragment_bson_disk(chunk_size, acquire)
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
         
@@ -1795,7 +2010,7 @@ class Table:
                 self.release()
 
     ############### WRITE ###############
-    def write(self, force_write=False):
+    def write(self, force_write=False, strict=False):
         """
         Writes data to a file and uploads it concurrently using separate threads.
         
@@ -1811,19 +2026,47 @@ class Table:
         7. Handles exceptions by logging an error message and re-raising the exception.
         8. Releases the lock regardless of success or failure.
         
+        A schemaless table whose BSON file is about to be uploaded is compacted first, in this thread
+        and under this lock (the compaction is a STRUCT publication, so it cannot run in the upload
+        thread), and before the header, checksums and timestamp are prepared, so that the uploaded head
+        and tail carry the moved pointers.
+
         Parameters:
             force_write (bool): If True, forces the header to be written even if not modified.
+            strict (bool): If True an upload or flush failure raises. It is always logged with the table name.
         
         Raises:
-            Exception: If the write or upload operation fails.
+            Exception: If the write fails (or, with strict, if the upload or the local flush fails).
         """
         errmsg = ''
         self.acquire()
         try:
 
             tini = time.time()
+            from SharedData.services import s3_enabled
+            remote = s3_enabled()
             # create header
             mtime = self.hdr['mtime']
+            compacted = False
+            if remote and self.is_schemaless and self.type == 1 and self.bsonpath.exists():
+                remote_bson_mtime = S3GetMtime(str(self.bsonpath) + '.gzip', self.database_folder)
+                if force_write or remote_bson_mtime is None or remote_bson_mtime < mtime:
+                    try:
+                        stats = self.compact_bson_if_worthwhile()
+                    except BsonIntegrityError as error:
+                        # Loud, every night, but the upload goes on as it was: nothing is rewritten.
+                        Logger.log.error('BSON compaction skipped: %s' % error)
+                        stats = None
+                    if stats is not None:
+                        compacted = True
+                        # New pointers: head, tail and BSON are all rewritten and all uploaded, under a
+                        # publication time later than anything the remote holds, so consumers see the change.
+                        newest = max([m for m in (remote_bson_mtime,
+                                                  S3GetMtime(str(self.headpath) + '.gzip', self.database_folder),
+                                                  S3GetMtime(str(self.tailpath) + '.gzip', self.database_folder))
+                                      if m is not None] + [float(self.hdr['mtime'])])
+                        self.hdr['mtime'] = max(float(self.hdr['mtime']), newest + 0.001)
+                        mtime = self.hdr['mtime']
             write_head = False            
             write_head = self.fill_header(force_write)
 
@@ -1833,21 +2076,32 @@ class Table:
                 if self.bson_file_handle is not None:
                     os.fsync(self.bson_file_handle.fileno())
 
-            from SharedData.services import s3_enabled
-            if not s3_enabled():
+            if not remote:
                 self.write_file()
                 return
 
+            failures = []
+
+            def guarded(label, target, *args):
+                # A thread's exception never reaches join(): keep it, name the table, let the caller decide.
+                try:
+                    target(*args)
+                except BaseException as error:
+                    failures.append('%s: %s' % (label, error))
+                    Logger.log.error('%s %s failed: %s' % (self.relpath, label, error))
+
             thread_s3 = threading.Thread(
-                target=self.upload, args=(write_head, mtime, force_write))
+                target=guarded, args=('upload', self.upload, write_head, mtime, force_write or compacted, compacted))
             thread_s3.start()
 
-            thread_flush = threading.Thread(target=self.write_file)
+            thread_flush = threading.Thread(target=guarded, args=('flush', self.write_file))
             thread_flush.start()
 
             # join threads
             thread_s3.join()
             thread_flush.join()
+            if failures and strict:
+                raise Exception('; '.join(failures))
 
             te = time.time() - tini
             datasize = self.hdr['count'] * self.hdr['itemsize'] / 1000000
@@ -2030,33 +2284,14 @@ class Table:
         return write_head
 
     ############### UPLOAD ###############
-    def upload(self, write_head, mtime, force_write=False):
+    def upload(self, write_head, mtime, force_write=False, force_tail=False):
         """
         Uploads the head, tail, and BSON files to remote storage based on modification times and flags.
         
-        For schemaless tables, defragments the BSON file FIRST to ensure that the uploaded head/tail
-        contain the correct updated pointers for the defragmented BSON file.
+        For schemaless tables the BSON file is uploaded after head and tail. Compaction is not done here:
+        it is a structural publication and runs in write(), before the header is prepared.
         """
-        # DEFRAGMENT BSON FIRST (before uploading head/tail) for schemaless tables
-        # This ensures the pointers in records are updated before we upload them
-        if self.is_schemaless and self.bsonpath.exists():
-            bson_gzip_path = str(self.bsonpath) + '.gzip'
-            remote_bson_mtime = S3GetMtime(bson_gzip_path, self.database_folder)
-            remote_bson_is_updated = False
-            if remote_bson_mtime is not None:
-                remote_bson_is_updated = remote_bson_mtime >= mtime
-            
-            # Defragment if BSON needs upload (this updates pointers in records)
-            if not remote_bson_is_updated or force_write:
-                # Pass acquire=False because we're already in write() context with lock held
-                stats = self.defragment_bson(acquire=False)
-                
-                # Mark header as modified to force head/tail upload with new pointers
-                write_head = True
-                self.hdr['mtimehead'] = mtime
-                self.hdr['mtimetail'] = mtime
-        
-        # Now upload head/tail with UPDATED pointers
+        # head/tail carry the pointers of the BSON uploaded below (compacted by write() when it was worth it)
         remote_head_mtime = S3GetMtime(str(self.headpath)+'.gzip', self.database_folder)
         remote_head_is_updated = False
         if not remote_head_mtime is None:
@@ -2070,7 +2305,7 @@ class Table:
             remote_tail_is_updated = False
             if not remote_tail_mtime is None:
                 remote_tail_is_updated = remote_tail_mtime >= self.hdr['mtimetail']
-            if not remote_tail_is_updated:
+            if force_tail or not remote_tail_is_updated:
                 self.upload_tail(mtime)
         else:
             self.delete_tail()
@@ -2741,6 +2976,8 @@ class Table:
         if (dirty or self._mapping_epoch != current
                 or self.size != int(self.hdr['recordssize'])):
             self.attach()
+            if getattr(self, 'is_schemaless', False):
+                self._reset_bson_mapping()      # the BSON file is published together with data.bin
             if self.hasindex and not dirty:
                 self.index.rememmap()
             self._mapping_epoch = current