shareddata 6.85.1__tar.gz → 6.85.3__tar.gz

{shareddata-6.85.1/src/shareddata.egg-info → shareddata-6.85.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.1
+Version: 6.85.3
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.1 → shareddata-6.85.3}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.1',
+    version='6.85.3',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.1 → shareddata-6.85.3}/src/SharedData/Table.py

@@ -865,6 +865,27 @@ class Table:
         raise RewriteError('automatic disk header migration requires staged publication')
 
     ############### DOWNLOAD ###############
+    def _remote_change(self):
+        """What a reload would have to fetch: 'head' when the remote head is newer than this table's
+        (the whole image changed), 'tail' when only the remote tail is (the usual case: the rows of
+        the current year), None when nothing is newer, there is no remote copy, or S3 is off."""
+        from SharedData.services import s3_enabled
+        if not s3_enabled() or self.hdr is None:
+            return None
+        head = S3GetMtime(str(self.headpath) + '.gzip', self.database_folder)
+        if head is not None and head > float(self.hdr['mtimehead']):
+            return 'head'
+        if int(self.hdr['hastail']) == 1:
+            tail = S3GetMtime(str(self.tailpath) + '.gzip', self.database_folder)
+            if tail is not None and tail > float(self.hdr['mtimetail']):
+                return 'tail'
+        return None
+
+    def _download_remote_tail(self):
+        """[io, local mtime, remote mtime] of the remote tail; io is None unless it is newer than this table's."""
+        return S3Download(str(self.tailpath) + '.gzip', local_mtime=float(self.hdr['mtimetail']),
+                          database_folder=self.database_folder)
+
     @structural
     def download(self):
         """

{shareddata-6.85.1 → shareddata-6.85.3}/src/SharedData/TableDisk.py

@@ -263,27 +263,126 @@ class TableDisk(Table):
     @structural
     def reload(self):
         """
-        Reloads the table data from disk by performing a sequence of operations to ensure data consistency and resource management.
-        
-        The method:
-        - Acquires necessary locks or resources to safely perform the reload.
-        - Frees current resources without acquiring new ones to prepare for fresh data.
-        - Downloads the latest table data from disk.
-        - Allocates memory for the newly downloaded data.
-        - If an index exists, resets its creation flag and initializes the index.
-        - Updates shareddata references to point to the reloaded table.
-        - Marks the table as loaded in both header and mutex states.
-        - Handles and logs any exceptions that occur during the reload process.
-        - Ensures that all acquired resources are released after the operation, regardless of success or failure.
-        
+        Replace the local image with the remote copy when that is newer.
+
+        Nothing newer remotely (or no remote copy, or S3 off) leaves the local table as it is, so
+        rows not uploaded yet are never discarded. When only the remote tail is newer (the usual
+        case: the head holds the years before this one and the tail this year's rows) only the tail
+        is fetched and applied, see _reload_tail. When the remote head is newer, the whole remote
+        image is built beside data.bin under a stage name and installed by one os.replace, inside
+        the caller's STRUCT section: a failure before the replace leaves the old image untouched, and the epoch
+        published on leaving the section makes other processes reopen the new file. The old index
+        files describe the old rows, so they are removed and rebuilt.
+
         Returns:
             The records of the reloaded table.
         """
-        from SharedData.TableRewrite import RewriteError
-        raise RewriteError('disk reload requires staged remote replacement')
+        import stat
+        import uuid
+        from SharedData.TableRewrite import fsync_directory, read_layout, require_rewrite_support, sync_file
+        change = self._remote_change()
+        if change is None:
+            return self.records
+        if change == 'tail' and not self.is_schemaless:
+            return self._reload_tail()
+        require_rewrite_support()
+        final = self.filepath
+        old = os.stat(final)
+        for leftover in final.parent.glob('data.bin.reload.*'):     # the writer lock is held: a stage left by a dead reload
+            leftover.unlink()
+        stage = final.parent / ('data.bin.reload.' + uuid.uuid4().hex)
+        saved = (self.hdr, self.size, getattr(self, 'orig_hdr', None))
+        self.filepath, self.hdr = stage, None                       # None: download the remote image, not what is newer than the local one
+        self._bootstrap_record_download = True
+        try:
+            self._download_empty_image()
+            if self.hdr is None or not stage.is_file():
+                raise Exception('the remote copy of %s disappeared during the reload' % self.relpath)
+            sync_file(stage)
+            with open(stage, 'rb') as stream:
+                downloaded, _ = read_layout(stream)
+            os.chmod(stage, stat.S_IMODE(old.st_mode))
+            try:
+                os.chown(stage, old.st_uid, old.st_gid)
+            except OSError:
+                pass
+            os.replace(stage, final)
+        except BaseException:
+            self.filepath = final
+            self.hdr, self.size, self.orig_hdr = saved
+            try:
+                stage.unlink()
+            except FileNotFoundError:
+                pass
+            raise
+        finally:
+            self._bootstrap_record_download = False
+        self.filepath = final
+        fsync_directory(final.parent)
+        try:
+            self.free(acquire=False)                                 # the old mappings and index arrays, flushed first
+            for name in ('dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
+                try:
+                    (final.parent / name).unlink()
+                except FileNotFoundError:
+                    pass
+            self.malloc()
+            if self.hasindex:
+                self.hdr['isidxcreated'] = 0
+                self.index.initialize()
+            self.shareddata.data[self.relpath] = self
+            self.hdr['isloaded'] = 1
+            self.mutex['isloaded'] = 1
+        except Exception as e:
+            errmsg = f"TableDisk.reload() {self.relpath}: {e}"
+            Logger.log.error(errmsg)
+            raise Exception(errmsg)
+        return self.records
+
+    def _reload_tail(self):
+        """
+        Apply the remote tail alone: its cost is the tail, not the table (that is what the head/tail
+        split is for). Runs inside reload's STRUCT section, with the writer lock held.
+
+        The tail rows are written into the live file right after the head, the file is synced, and
+        the header (count, tail size and time) is committed last. A process that dies before the
+        commit leaves the old header: the old tail time is still older than the remote one, so the
+        next reload applies the tail again, and the index is rebuilt from the committed rows (the
+        dead holder leaves the table flagged DIRTY). No reader looks at the file meanwhile: STRUCT
+        keeps lock-free readers waiting and the other readers take the same lock.
+        """
+        from SharedData.TableRewrite import sync_file
+        tail_gz, _, remote_mtime = self._download_remote_tail()
+        if tail_gz is None:                                           # nothing newer after all
+            return self.records
+        live, size = self.hdr, self.size
+        self.hdr = live.copy()                                        # the mapped header stays as it was until the commit
+        self._bootstrap_record_download = True                        # reload holds the lock: the tail may be written into the live file
+        try:
+            tail_gz.seek(0)                                           # S3Download leaves the buffer at its end
+            tail_io = gzip.GzipFile(fileobj=tail_gz, mode='rb')
+            self.read_tailheader(tail_io)                             # count, tail size and tail time of the new header
+            self.hdr['mtimetail'] = remote_mtime
+            if int(self.hdr['count']) > int(self.hdr['recordssize']):
+                self.hdr['recordssize'] = self.hdr['count']
+            if self.size is None or self.size < int(self.hdr['recordssize']):
+                self.size = int(self.hdr['recordssize'])
+            self.create_file()                                        # a table that grew gets a longer file: never shortened or rewritten
+            self.read_tail(tail_io)
+            sync_file(self.filepath)
+            self.hdr['isloaded'] = 0
+            self.hdr['isidxcreated'] = 0
+            committed = self.hdr
+        except BaseException:
+            self.hdr, self.size = live, size
+            raise
+        finally:
+            self._bootstrap_record_download = False
+        self.shf_hdr[0] = committed                                   # the commit
+        self.shf_hdr.flush()
+        sync_file(self.filepath)
         try:
             self.free(acquire=False)
-            self.download()
             self.malloc()
             if self.hasindex:
                 self.hdr['isidxcreated'] = 0
@@ -296,7 +395,7 @@ class TableDisk(Table):
             Logger.log.error(errmsg)
             raise Exception(errmsg)
         return self.records
-    
+
     ############### DELETE ###############
     def delete_local(self):    
         """

{shareddata-6.85.1 → shareddata-6.85.3/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.1
+Version: 6.85.3
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata