shareddata 6.85.2__tar.gz → 6.85.3__tar.gz

{shareddata-6.85.2/src/shareddata.egg-info → shareddata-6.85.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.2
+Version: 6.85.3
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.2 → shareddata-6.85.3}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.2',
+    version='6.85.3',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.2 → shareddata-6.85.3}/src/SharedData/Table.py

@@ -865,20 +865,26 @@ class Table:
         raise RewriteError('automatic disk header migration requires staged publication')
 
     ############### DOWNLOAD ###############
-    def _remote_is_newer(self):
-        """True when the remote head or tail is newer than this table's (what S3Download would
-        fetch). Nothing newer, no remote copy, or S3 switched off: False, and the local image stays."""
+    def _remote_change(self):
+        """What a reload would have to fetch: 'head' when the remote head is newer than this table's
+        (the whole image changed), 'tail' when only the remote tail is (the usual case: the rows of
+        the current year), None when nothing is newer, there is no remote copy, or S3 is off."""
         from SharedData.services import s3_enabled
         if not s3_enabled() or self.hdr is None:
-            return False
+            return None
         head = S3GetMtime(str(self.headpath) + '.gzip', self.database_folder)
         if head is not None and head > float(self.hdr['mtimehead']):
-            return True
+            return 'head'
         if int(self.hdr['hastail']) == 1:
             tail = S3GetMtime(str(self.tailpath) + '.gzip', self.database_folder)
             if tail is not None and tail > float(self.hdr['mtimetail']):
-                return True
-        return False
+                return 'tail'
+        return None
+
+    def _download_remote_tail(self):
+        """[io, local mtime, remote mtime] of the remote tail; io is None unless it is newer than this table's."""
+        return S3Download(str(self.tailpath) + '.gzip', local_mtime=float(self.hdr['mtimetail']),
+                          database_folder=self.database_folder)
 
     @structural
     def download(self):

{shareddata-6.85.2 → shareddata-6.85.3}/src/SharedData/TableDisk.py

@@ -266,9 +266,11 @@ class TableDisk(Table):
         Replace the local image with the remote copy when that is newer.
 
         Nothing newer remotely (or no remote copy, or S3 off) leaves the local table as it is, so
-        rows not uploaded yet are never discarded. Otherwise the whole remote image is built beside
-        data.bin under a stage name and installed by one os.replace, inside the caller's STRUCT
-        section: a failure before the replace leaves the old image untouched, and the epoch
+        rows not uploaded yet are never discarded. When only the remote tail is newer (the usual
+        case: the head holds the years before this one and the tail this year's rows) only the tail
+        is fetched and applied, see _reload_tail. When the remote head is newer, the whole remote
+        image is built beside data.bin under a stage name and installed by one os.replace, inside
+        the caller's STRUCT section: a failure before the replace leaves the old image untouched, and the epoch
         published on leaving the section makes other processes reopen the new file. The old index
         files describe the old rows, so they are removed and rebuilt.
 
@@ -278,8 +280,11 @@ class TableDisk(Table):
         import stat
         import uuid
         from SharedData.TableRewrite import fsync_directory, read_layout, require_rewrite_support, sync_file
-        if not self._remote_is_newer():
+        change = self._remote_change()
+        if change is None:
             return self.records
+        if change == 'tail' and not self.is_schemaless:
+            return self._reload_tail()
         require_rewrite_support()
         final = self.filepath
         old = os.stat(final)
@@ -334,6 +339,63 @@ class TableDisk(Table):
             raise Exception(errmsg)
         return self.records
 
+    def _reload_tail(self):
+        """
+        Apply the remote tail alone: its cost is the tail, not the table (that is what the head/tail
+        split is for). Runs inside reload's STRUCT section, with the writer lock held.
+
+        The tail rows are written into the live file right after the head, the file is synced, and
+        the header (count, tail size and time) is committed last. A process that dies before the
+        commit leaves the old header: the old tail time is still older than the remote one, so the
+        next reload applies the tail again, and the index is rebuilt from the committed rows (the
+        dead holder leaves the table flagged DIRTY). No reader looks at the file meanwhile: STRUCT
+        keeps lock-free readers waiting and the other readers take the same lock.
+        """
+        from SharedData.TableRewrite import sync_file
+        tail_gz, _, remote_mtime = self._download_remote_tail()
+        if tail_gz is None:                                           # nothing newer after all
+            return self.records
+        live, size = self.hdr, self.size
+        self.hdr = live.copy()                                        # the mapped header stays as it was until the commit
+        self._bootstrap_record_download = True                        # reload holds the lock: the tail may be written into the live file
+        try:
+            tail_gz.seek(0)                                           # S3Download leaves the buffer at its end
+            tail_io = gzip.GzipFile(fileobj=tail_gz, mode='rb')
+            self.read_tailheader(tail_io)                             # count, tail size and tail time of the new header
+            self.hdr['mtimetail'] = remote_mtime
+            if int(self.hdr['count']) > int(self.hdr['recordssize']):
+                self.hdr['recordssize'] = self.hdr['count']
+            if self.size is None or self.size < int(self.hdr['recordssize']):
+                self.size = int(self.hdr['recordssize'])
+            self.create_file()                                        # a table that grew gets a longer file: never shortened or rewritten
+            self.read_tail(tail_io)
+            sync_file(self.filepath)
+            self.hdr['isloaded'] = 0
+            self.hdr['isidxcreated'] = 0
+            committed = self.hdr
+        except BaseException:
+            self.hdr, self.size = live, size
+            raise
+        finally:
+            self._bootstrap_record_download = False
+        self.shf_hdr[0] = committed                                   # the commit
+        self.shf_hdr.flush()
+        sync_file(self.filepath)
+        try:
+            self.free(acquire=False)
+            self.malloc()
+            if self.hasindex:
+                self.hdr['isidxcreated'] = 0
+                self.index.initialize()
+            self.shareddata.data[self.relpath] = self
+            self.hdr['isloaded'] = 1
+            self.mutex['isloaded'] = 1
+        except Exception as e:
+            errmsg = f"TableDisk.reload() {self.relpath}: {e}"
+            Logger.log.error(errmsg)
+            raise Exception(errmsg)
+        return self.records
+
     ############### DELETE ###############
     def delete_local(self):    
         """

{shareddata-6.85.2 → shareddata-6.85.3/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.2
+Version: 6.85.3
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata