shareddata 6.85.3__tar.gz → 6.85.5__tar.gz

{shareddata-6.85.3/src/shareddata.egg-info → shareddata-6.85.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.3
+Version: 6.85.5
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.3 → shareddata-6.85.5}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.3',
+    version='6.85.5',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.3 → shareddata-6.85.5}/src/SharedData/Table.py

@@ -1229,6 +1229,17 @@ class Table:
                     io_obj.close()
                 self.copy_file()
 
+    def _parse_tail_header(self, tail_io):
+        """The tail header record at the start of a tail stream (its first row starts right after it).
+        Unlike read_tailheader, nothing of this table is changed."""
+        tail_io.seek(0)
+        size = int.from_bytes(tail_io.read(8), byteorder='little')
+        description = tail_io.read(size).decode(encoding='UTF-8', errors='ignore').replace('\x00', '')
+        names, formats = description.split(';')[0].split(','), description.split(';')[1].split(',')
+        dtype = np.dtype({'names': names, 'formats': formats})
+        tail_io.seek(0)
+        return np.ndarray((1,), dtype=dtype, buffer=tail_io.read(dtype.itemsize))[0].copy()
+
     def read_tailheader(self, tail_io):
         """
         Reads and parses the tail header from a binary stream.

{shareddata-6.85.3 → shareddata-6.85.5}/src/SharedData/TableDisk.py

@@ -23,7 +23,51 @@ from SharedData.Table import Table
 from SharedData.LockWaiters import TableBusy
 
 from SharedData.Utils import cpp
-from SharedData.TableProtocol import structural
+from SharedData.TableProtocol import structural, structure
+
+
+FICLONE = 0x40049409          # linux/fs.h: make the file a reflink clone of another
+CHUNK = 64 << 20
+
+
+def _clone_prefix(source, stage, prefix):
+    """Create *stage* as a copy of the first *prefix* bytes of *source*, which is only read.
+
+    On a filesystem with reflinks (XFS, btrfs) the whole file is cloned: no data is read or
+    written and no space is used until one of them changes. Anywhere else just the prefix (the
+    header and the head) is copied by the kernel. Returns 'reflink' or 'copy'.
+    """
+    import errno
+    import fcntl
+    from SharedData.TableRewrite import RewriteError
+    unsupported = (errno.EOPNOTSUPP, errno.ENOTTY, errno.EXDEV, errno.EINVAL, errno.ENOSYS)
+    src, dst = os.open(source, os.O_RDONLY), None
+    try:
+        dst = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+        try:
+            fcntl.ioctl(dst, FICLONE, src)
+            return 'reflink'
+        except OSError as error:
+            if error.errno not in unsupported:
+                raise
+        copied = 0
+        while copied < prefix:
+            want = min(1 << 30, prefix - copied)
+            try:
+                done = os.copy_file_range(src, dst, want, copied, copied)
+            except OSError as error:
+                if error.errno not in unsupported:
+                    raise
+                chunk = os.pread(src, min(want, CHUNK), copied)
+                done = os.pwrite(dst, chunk, copied) if chunk else 0
+            if done <= 0:
+                raise RewriteError('the live image ended before its head was copied')
+            copied += done
+        return 'copy'
+    finally:
+        os.close(src)
+        if dst is not None:
+            os.close(dst)
 
 
 class TableDisk(Table):    
@@ -260,7 +304,6 @@ class TableDisk(Table):
         return publish_rewrite(self, None, 'trim', capacity=max(2, int(self.hdr['count'])))
 
     ############### RELOAD ###############
-    @structural
     def reload(self):
         """
         Replace the local image with the remote copy when that is newer.
@@ -269,14 +312,28 @@ class TableDisk(Table):
         rows not uploaded yet are never discarded. When only the remote tail is newer (the usual
         case: the head holds the years before this one and the tail this year's rows) only the tail
         is fetched and applied, see _reload_tail. When the remote head is newer, the whole remote
-        image is built beside data.bin under a stage name and installed by one os.replace, inside
-        the caller's STRUCT section: a failure before the replace leaves the old image untouched, and the epoch
-        published on leaving the section makes other processes reopen the new file. The old index
-        files describe the old rows, so they are removed and rebuilt.
+        image is fetched, see _reload_image.
 
         Returns:
             The records of the reloaded table.
         """
+        change = self._remote_change()
+        if change is None:
+            return self.records
+        if change == 'tail' and not self.is_schemaless:
+            done = self._reload_tail()
+            if done is not None:
+                return done
+        return self._reload_image()
+
+    @structural
+    def _reload_image(self):
+        """
+        The whole remote image is built beside data.bin under a stage name and installed by one
+        os.replace, inside a STRUCT section: a failure before the replace leaves the old image
+        untouched, and the epoch published on leaving the section makes other processes reopen the
+        new file. The old index files describe the old rows, so they are removed and rebuilt.
+        """
         import stat
         import uuid
         from SharedData.TableRewrite import fsync_directory, read_layout, require_rewrite_support, sync_file
@@ -339,62 +396,146 @@ class TableDisk(Table):
             raise Exception(errmsg)
         return self.records
 
+    def _attach_replacement(self, final, rebuild_files):
+        """After os.replace: make the directory entry durable, then map the new image and rebuild its
+        indexes. Whatever fails, this object ends mapped to the image that is now installed (flagged
+        for index repair by the replacement's own header), never to two images."""
+        from SharedData.TableRewrite import fsync_directory
+        try:
+            fsync_directory(final.parent)                           # the new name is durable before any old index file is touched
+            self.free(acquire=False)                                # the old mappings and index arrays, flushed first
+            if rebuild_files:
+                for name in ('dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
+                    try:
+                        (final.parent / name).unlink()
+                    except FileNotFoundError:
+                        pass
+            self.malloc()
+            if self.hasindex:
+                self.hdr['isidxcreated'] = 0
+                self.index.initialize()
+            self.shareddata.data[self.relpath] = self
+            self.hdr['isloaded'] = 1
+            self.mutex['isloaded'] = 1
+        except BaseException as error:
+            self._mapping_epoch = -1                                # the next owner re-attaches by path
+            try:
+                self.attach()
+            except Exception:
+                pass
+            if isinstance(error, Exception) and not isinstance(error, OSError):
+                Logger.log.error(f"TableDisk.reload() {self.relpath}: {error}")
+            raise
+        return self.records
+
     def _reload_tail(self):
         """
-        Apply the remote tail alone: its cost is the tail, not the table (that is what the head/tail
-        split is for). Runs inside reload's STRUCT section, with the writer lock held.
+        Apply the remote tail alone: only the tail object is downloaded (that is what the head/tail
+        split is for) and no live file is written (see TableRewrite).
 
-        The tail rows are written into the live file right after the head, the file is synced, and
-        the header (count, tail size and time) is committed last. A process that dies before the
-        commit leaves the old header: the old tail time is still older than the remote one, so the
-        next reload applies the tail again, and the index is rebuilt from the committed rows (the
-        dead holder leaves the table flagged DIRTY). No reader looks at the file meanwhile: STRUCT
-        keeps lock-free readers waiting and the other readers take the same lock.
+        Under the writer lock only (readers go on reading the live image meanwhile) a stage is built
+        as a clone of the live image, see _clone_prefix: the head, which did not change, is shared or
+        copied by the kernel. Its unused capacity is allocated again, since a clone shares only what
+        was written. The tail is streamed into the stage with a bounded buffer, its length and checksum
+        checked against its header, then the new header (indexes marked not created) is written, the
+        stage synced. A failure up to here leaves the live image and header untouched and the table
+        not flagged: nothing was changed. Then, in a STRUCT section, one os.replace installs the stage,
+        the directory is synced before any index file is touched, the indexes are rebuilt and the epoch
+        published. A process that dies at any point leaves the complete old image or the complete new
+        one, and nothing is needed from S3 to recover. A remote head that changed meanwhile means the
+        tail belongs to another head: the whole image is replaced instead (returns None).
         """
-        from SharedData.TableRewrite import sync_file
-        tail_gz, _, remote_mtime = self._download_remote_tail()
-        if tail_gz is None:                                           # nothing newer after all
-            return self.records
-        live, size = self.hdr, self.size
-        self.hdr = live.copy()                                        # the mapped header stays as it was until the commit
-        self._bootstrap_record_download = True                        # reload holds the lock: the tail may be written into the live file
+        import uuid
+        from SharedData.TableRewrite import (RewriteError, copy_metadata, pwrite_all, require_rewrite_support,
+                                             same_digest, sync_header)
+        import hashlib
+        require_rewrite_support()
+        acquired = not self._owns_lock()
+        if acquired:
+            self.acquire()
+        stage = None
         try:
+            self._sync_held_mapping()
+            tail_gz, _, remote_mtime = self._download_remote_tail()
+            if tail_gz is None:                                       # nothing newer after all
+                return self.records
+            # The tail belongs to the head it was uploaded with, and the uploader sends the head first:
+            # a head that changed since the check in reload() means this tail is not for the head held here.
+            if self._remote_change() == 'head':
+                return None
+            final = self.filepath
+            header = self.hdr.copy()                                  # detached: this object's header is not touched
             tail_gz.seek(0)                                           # S3Download leaves the buffer at its end
             tail_io = gzip.GzipFile(fileobj=tail_gz, mode='rb')
-            self.read_tailheader(tail_io)                             # count, tail size and tail time of the new header
-            self.hdr['mtimetail'] = remote_mtime
-            if int(self.hdr['count']) > int(self.hdr['recordssize']):
-                self.hdr['recordssize'] = self.hdr['count']
-            if self.size is None or self.size < int(self.hdr['recordssize']):
-                self.size = int(self.hdr['recordssize'])
-            self.create_file()                                        # a table that grew gets a longer file: never shortened or rewritten
-            self.read_tail(tail_io)
-            sync_file(self.filepath)
-            self.hdr['isloaded'] = 0
-            self.hdr['isidxcreated'] = 0
-            committed = self.hdr
+            tailhdr = self._parse_tail_header(tail_io)
+            itemsize, tailsize = int(header['itemsize']), int(tailhdr['tailsize'])
+            header['md5hashtail'] = tailhdr['md5hash']
+            header['tailsize'] = tailsize
+            header['count'] = int(header['headsize']) + tailsize
+            header['mtimetail'] = remote_mtime
+            if int(header['count']) > int(header['recordssize']):
+                header['recordssize'] = header['count']
+            capacity = int(header['recordssize'])
+            hdrsize = header.dtype.itemsize
+            tail_at = hdrsize + int(header['headsize']) * itemsize
+            total = hdrsize + capacity * itemsize
+            header['isidxcreated'] = 0                                # durable in the stage: no client may trust the old indexes
+            header['isloaded'] = 0
+            header['isidxsorted'] = 1                                 # the uploader sorts before it writes (SharedNumpy.write)
+            pending = int(header['minchgid'])                         # a head row edited here and not uploaded yet stays marked for upload;
+            header['minchgid'] = pending if pending <= int(header['headsize']) else int(header['count']) + 1   # otherwise as a write leaves it
+            for leftover in final.parent.glob('data.bin.reload.*'):   # the lock is held: a stage left by a dead reload
+                leftover.unlink()
+            stage = final.parent / ('data.bin.reload.' + uuid.uuid4().hex)
+            self.shf_data.flush()
+            sync_header(self)                                         # the old image is complete on disk before it is cloned
+            _clone_prefix(final, stage, tail_at)
+            fd = os.open(stage, os.O_RDWR)
+            try:
+                copy_metadata(final, fd)
+                os.posix_fallocate(fd, 0, total)                      # a clone shares written extents only: the capacity is reserved again
+                tail_io.seek(tailhdr.dtype.itemsize)
+                digest, written = hashlib.md5(), 0
+                while True:
+                    chunk = tail_io.read(CHUNK)
+                    if not chunk:
+                        break
+                    if written + len(chunk) > tailsize * itemsize:
+                        raise RewriteError('the remote tail of %s is longer than its header says' % self.relpath)
+                    digest.update(chunk)
+                    pwrite_all(fd, chunk, tail_at + written)
+                    written += len(chunk)
+                if written != tailsize * itemsize:
+                    raise RewriteError('the remote tail of %s holds %d bytes, its header says %d' % (self.relpath, written, tailsize * itemsize))
+                if not same_digest(digest.digest(), tailhdr['md5hash']):
+                    raise RewriteError('the remote tail of %s does not match its checksum' % self.relpath)
+                pwrite_all(fd, header.tobytes(), 0)
+                os.fsync(fd)
+            finally:
+                os.close(fd)
+            refused = None
+            with structure(self):                                     # only the swap and the index rebuild block readers
+                try:
+                    os.replace(stage, final)
+                except OSError as error:                              # nothing changed: leave STRUCT without flagging the live header
+                    refused = error
+                else:
+                    stage = None
+                    self.size = capacity
+                    self._attach_replacement(final, rebuild_files=True)
+            if refused is not None:
+                raise refused
+            return self.records
         except BaseException:
-            self.hdr, self.size = live, size
+            if stage is not None:
+                try:
+                    stage.unlink()
+                except FileNotFoundError:
+                    pass
             raise
         finally:
-            self._bootstrap_record_download = False
-        self.shf_hdr[0] = committed                                   # the commit
-        self.shf_hdr.flush()
-        sync_file(self.filepath)
-        try:
-            self.free(acquire=False)
-            self.malloc()
-            if self.hasindex:
-                self.hdr['isidxcreated'] = 0
-                self.index.initialize()
-            self.shareddata.data[self.relpath] = self
-            self.hdr['isloaded'] = 1
-            self.mutex['isloaded'] = 1
-        except Exception as e:
-            errmsg = f"TableDisk.reload() {self.relpath}: {e}"
-            Logger.log.error(errmsg)
-            raise Exception(errmsg)
-        return self.records
+            if acquired:
+                self.release()
 
     ############### DELETE ###############
     def delete_local(self):    

{shareddata-6.85.3 → shareddata-6.85.5}/src/SharedData/TableRewrite.py

@@ -61,6 +61,32 @@ def write_all(fd, data):
         data = data[written:]
 
 
+def pwrite_all(fd, data, offset):
+    """os.pwrite until every byte is written; a write that makes no progress is an error."""
+    view = memoryview(data).cast('B')
+    done = 0
+    while done < len(view):
+        written = os.pwrite(fd, view[done:], offset + done)
+        if written <= 0:
+            raise RewriteError('no progress writing the replacement image')
+        done += written
+
+
+def copy_metadata(source, fd):
+    """Mode, owner, ACLs and extended attributes of *source* onto the open replacement *fd*
+    (fchmod is not filtered by the umask). A failure aborts the replacement, as in publish_rewrite."""
+    info = os.stat(source)
+    os.fchown(fd, info.st_uid, info.st_gid)
+    os.fchmod(fd, stat.S_IMODE(info.st_mode))
+    for attribute in os.listxattr(source):
+        os.setxattr(fd, attribute, os.getxattr(source, attribute))
+
+
+def same_digest(digest, stored):
+    """A header keeps an MD5 in a bytes field, which NumPy returns without its trailing NULs."""
+    return bytes(stored) == digest.rstrip(b'\0')
+
+
 def read_layout(stream):
     """Strict, read-only self-described header parser, also used before open."""
     stream.seek(0)

{shareddata-6.85.3 → shareddata-6.85.5}/src/SharedData/TimeSeriesDisk.py

@@ -124,8 +124,7 @@ class TimeSeriesDisk:
 
             if value is not None:
                 # mutex() acquired the non-reentrant tag lock for initialization.
-                self._write_values_locked(value.index, value.columns, value.to_numpy(),
-                                          add_columns=True, flush=True)
+                self._write_constructor_value_locked(value)
         finally:
             self.release()
 
@@ -188,6 +187,23 @@ class TimeSeriesDisk:
         finally:
             self.release()
 
+    def _write_constructor_value_locked(self, value):
+        """
+        The `value=` of the constructor, with the meaning it always had: rows whose timestamp is not
+        a slot of the series (a weekend, a date after its end) are skipped, not an error. Routines
+        build the frame from other tables and hand it over as is. The exact-timestamp check belongs to
+        write_values(), the explicit bulk API.
+        """
+        self.ensure_columns(value.columns, _locked=True)
+        tidx = self.get_loc_timestamp(value.index.values.astype(np.int64) / 10**9)
+        skipped = int(np.isnan(tidx).sum())
+        if skipped:
+            Logger.log.warning('%s: %d of %d rows of value are not on the series time index and were skipped'
+                               % (self.relpath, skipped, len(tidx)))
+        sidx = self.columns.get_indexer(value.columns).astype(np.float64)
+        self.setValuesJit(self.shf, tidx, sidx, np.asarray(value.to_numpy(), dtype=np.float64))
+        self.shf.flush()
+
     def _write_values_locked(self, index, columns, values, *, add_columns, flush):
         index = pd.DatetimeIndex(index)
         columns = pd.Index(columns)

{shareddata-6.85.3 → shareddata-6.85.5/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.3
+Version: 6.85.5
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata