shareddata 6.85.11__tar.gz → 6.85.14__tar.gz

{shareddata-6.85.11/src/shareddata.egg-info → shareddata-6.85.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.11
+Version: 6.85.14
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.11 → shareddata-6.85.14}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.11',
+    version='6.85.14',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.11 → shareddata-6.85.14}/src/SharedData/Routines/WorkerLib.py

@@ -915,7 +915,7 @@ def _install_repo(command, quiet=False):
                 if not send_command(checkout_cmd):
                     Logger.log.error(f'Checking out branch {command["branch"]} FAILED!')
                     runroutine = False
-                else:
+                elif not quiet:
                     Logger.log.info(f'Checked out branch {command["branch"]}')
             else:
                 # Checkout main branch if no branch specified
@@ -923,7 +923,7 @@ def _install_repo(command, quiet=False):
                 if not send_command(checkout_cmd):
                     Logger.log.error(f'Checking out branch main FAILED!')
                     runroutine = False
-                else:
+                elif not quiet:
                     Logger.log.info(f'Checked out branch main')
 
             # pull existing repo

{shareddata-6.85.11 → shareddata-6.85.14}/src/SharedData/SharedData.py

@@ -6,6 +6,7 @@ import json
 import warnings
 import time
 import shutil
+from SharedData.TableRewrite import remove_with_retry, rmtree_with_retry
 from datetime import datetime, timezone
 
 from multiprocessing import shared_memory
@@ -989,7 +990,7 @@ class SharedData:
                 fpath = Path(os.environ['DATABASE_FOLDER'])
                 fpath = fpath/('shm/'+shm_name.replace('\\', '/')+'.csv')
                 if fpath.is_file():
-                    os.remove(fpath)
+                    remove_with_retry(fpath)
             except:
                 pass
 
@@ -1685,7 +1686,7 @@ class SharedData:
                     except:
                         try:
                             if fpath.is_file():
-                                os.remove(fpath)
+                                remove_with_retry(fpath)
                         except:
                             pass
         shm_names = shm_names.sort_index()
@@ -1845,11 +1846,11 @@ class SharedData:
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                 # if folder is empty remove it
                 if localpath.exists():
                     if not any(localpath.iterdir()):
-                        shutil.rmtree(localpath)
+                        rmtree_with_retry(localpath)
             else:           
                 database_folder = os.environ['DATABASE_FOLDER']     
                 # select the first disk with less percent_used
@@ -1865,10 +1866,10 @@ class SharedData:
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                     # if folder is empty remove it
                     if not any(localpath.iterdir()):
-                        shutil.rmtree(localpath)
+                        rmtree_with_retry(localpath)
                 
             from SharedData.services import s3_enabled
             if not localonly and s3_enabled():
@@ -1911,12 +1912,12 @@ class SharedData:
                     del self.data[path]                
                 localpath = Path(os.environ['DATABASE_FOLDER'])/Path(path)
                 if localpath.exists():
-                    shutil.rmtree(localpath)
+                    rmtree_with_retry(localpath)
                 files_del = ['timeseries_head.bin','timeseries_tail.bin']
                 for file in files_del:
                     delpath = Path(localpath.parent/file)
                     if delpath.exists():
-                        os.remove(delpath)
+                        remove_with_retry(delpath)
                 from SharedData.services import s3_enabled
                 if not localonly and s3_enabled():
                     S3DeleteTimeseries(path.replace('/timeseries',''))
@@ -1928,7 +1929,7 @@ class SharedData:
                 fpath, shm_name = tstag.get_path()
                 del self.data[path].tags[tag]
                 del ts
-                os.remove(fpath)
+                remove_with_retry(fpath)
                 return True
             
         except Exception as e:

{shareddata-6.85.11 → shareddata-6.85.14}/src/SharedData/SharedNumpy.py

@@ -297,8 +297,9 @@ class SharedNumpy(np.ndarray):
             force_write (bool): If True, forces the write operation even if not necessary. Defaults to False.
         """
         # Saving remains available without unsupported record replacement.
-        if (self.table.type != 1 or sys.platform == 'linux') and (
-                self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):
+        from SharedData.TableRewrite import rewrite_supported
+        if (self.table.type != 1 or sys.platform == 'linux' or not rewrite_supported()) and (
+                self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):      # Windows sorts in place
             self.sort_index()
         self.table.write(force_write, strict)
         

{shareddata-6.85.11 → shareddata-6.85.14}/src/SharedData/Table.py

@@ -689,14 +689,25 @@ class Table:
         from SharedData.TableRewrite import (RewriteError, fsync_directory,
                                              rewrite_supported)
         if not rewrite_supported():
-            # No POSIX rename-over-a-mapped-file: remove the old files (the shared mutex is held in STRUCT
-            # mode, so this process is their only user; another process that still maps one makes the
-            # unlink fail loudly) and create the empty image in place.
-            for name in ('data.bin', 'bson.bin', 'dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
-                try:
-                    (self.filepath.parent / name).unlink()
-                except FileNotFoundError:
-                    pass
+            # Windows cannot rename over or unlink a file any process maps. The shared mutex is held in STRUCT
+            # mode, so the image is re-initialised in place: the old committed rows and the header are zeroed
+            # through a mapped view (coherent with every other process's view), the file is only ever extended,
+            # and the index / BSON files stay as they are (the index is rebuilt by the caller, and the new
+            # header resets next_bson_ptr, so no old document is reachable).
+            from SharedData.TableRewrite import RewriteError, extend_file, read_layout
+            final = self.filepath
+            try:
+                with open(final, 'rb') as stream:
+                    old_header, old_records = read_layout(stream)
+                clear = old_header.dtype.itemsize + int(old_header['count']) * old_records.itemsize
+            except RewriteError:
+                clear = os.path.getsize(final)          # not a coherent image: clear everything it holds
+            extend_file(final, int(self.hdrdtype.itemsize + self.recdtype.itemsize * int(self.size)))
+            clear = max(clear, int(self.hdrdtype.itemsize))
+            view = np.memmap(final, np.uint8, 'r+', 0, (clear,))
+            view[:] = 0
+            view.flush()
+            del view
             self.malloc(create=True)
             return
         final = self.filepath
@@ -1395,7 +1406,7 @@ class Table:
         return h1[:l] == h2[:l]
 
     ############### BSON MANAGEMENT ###############
-    def _reset_bson_mapping(self):
+    def _reset_bson_mapping(self, reopen=True):
         """Drop the BSON mapping and reopen the file by path: after a paired publication the old
         mapping belongs to an unlinked inode and its offsets no longer match the records."""
         if self.bson_mmap is not None:
@@ -1409,7 +1420,7 @@ class Table:
             self.bson_file_handle = None
         self.curr_bson_size = 0
         self.prev_bson_ptr = 0
-        if self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
+        if reopen and self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
             self.init_bson_mmap()
 
     def init_bson_mmap(self, from_download=False):
@@ -1669,10 +1680,6 @@ class Table:
 
     def compact_bson_if_worthwhile(self):
         """Compact the BSON file when enough of it is orphaned; None when it is not worth a rewrite."""
-        if self.type == 1:
-            from SharedData.TableRewrite import rewrite_supported
-            if not rewrite_supported():
-                return None         # the pair publication needs POSIX file APIs; compaction only saves space
         usage = self.bson_usage()
         if usage['orphaned'] < BSON_DEFRAG_MIN_BYTES \
                 or usage['orphaned'] < BSON_DEFRAG_MIN_FRACTION * max(usage['used'], 1):
@@ -1682,6 +1689,50 @@ class Table:
             return None
         return self.defragment_bson(acquire=False)
 
+    def _defrag_in_place(self, stage, mapping, payload, new_size):
+        """Windows: the compacted BSON `stage` is copied over the start of the live file and every moved
+        bson_ptr is rewritten in place, as one STRUCT section under the shared mutex (nothing else touches the
+        table), then the file is shrunk when no process maps it. Not crash-safe (a crash between the BSON
+        copy and the pointer update leaves STRUCT/DIRTY and pointers into a rewritten file); the staged
+        publication that covers that needs Linux."""
+        from SharedData.TableProtocol import structure
+        from SharedData.TableRewrite import shrink_file, sync_header
+        count = int(self.hdr['count'])
+        with structure(self):
+            if self.bson_mmap is not None:
+                self.bson_mmap.flush()
+            data = np.fromfile(stage, dtype=np.uint8)          # the stage is the compacted payload (+1 pad byte)
+            view = np.memmap(self.bsonpath, np.uint8, 'r+', 0, (len(data),))
+            view[:] = data
+            view.flush()
+            del view
+            rows = self.shf_data[:count]                       # the file's own mapping, never a caller-visible copy
+            selected = mapping >= 0
+            if selected.any():
+                column = rows['bson_ptr']
+                if column.dtype.kind == 'V' and column.dtype.itemsize == 8:
+                    column = column.view(np.int64)
+                column[selected] = mapping[selected]
+            self.shf_data.flush()
+            self.hdr['next_bson_ptr'] = payload
+            self.hdr['mtime'] = max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)
+            sync_header(self)
+            self.attach()                                      # records/header of this object: the mapping of the file, as after a publication
+            self._reset_bson_mapping(reopen=False)             # drop this process's map, then try to give the space back
+            physical = self.bsonpath.stat().st_size
+            if not shrink_file(self.bsonpath, new_size) and physical > new_size:
+                # another process still maps the file: it stays as large, but the stale documents behind the
+                # new end are zeroed so the upload (which compresses the file to its end) stays small
+                tail = np.memmap(self.bsonpath, np.uint8, 'r+', new_size, (physical - new_size,))
+                tail[:] = 0
+                tail.flush()
+                del tail
+            self._reset_bson_mapping()
+        try:
+            Path(stage).unlink()
+        except OSError:
+            pass
+
     def _defragment_bson_disk(self, chunk_size, acquire):
         """Compact bson.bin of a disk table and move every bson_ptr, as ONE crash-safe publication.
 
@@ -1695,7 +1746,8 @@ class Table:
         from SharedData.TableRewrite import (RewriteError, copy_metadata, file_identity, fsync_directory,
                                              pending_path, publish_rewrite, require_rewrite_support, write_all)
         from SharedData.TableProtocol import structure
-        require_rewrite_support()
+        from SharedData.TableRewrite import rewrite_supported
+        in_place = not rewrite_supported()
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
         if acquire:
@@ -1709,8 +1761,8 @@ class Table:
             if self.bson_mmap is not None:
                 self.bson_mmap.flush()
             if self.bsonpath.exists():
-                with open(self.bsonpath, 'rb') as baseline:
-                    os.fsync(baseline.fileno())
+                from SharedData.TableRewrite import sync_file
+                sync_file(self.bsonpath)                    # r+b on Windows: fsync needs a writable handle
             count = int(self.hdr['count'])
             pointers, sizes = self._bson_columns(count) if count else (np.zeros(0, np.int64), np.zeros(0, np.int64))
             live_idx = np.flatnonzero(sizes > 0)
@@ -1736,9 +1788,10 @@ class Table:
             token = uuid.uuid4().hex
             stage = self.bsonpath.parent / ('bson.bin.stage.' + token)
             digest = hashlib.sha256()
-            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0), 0o600)
             try:
-                copy_metadata(self.bsonpath, fd)
+                if not in_place:
+                    copy_metadata(self.bsonpath, fd)
                 if len(live_idx):
                     with open(self.bsonpath, 'rb') as source:
                         view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
@@ -1765,6 +1818,15 @@ class Table:
             new_size = max(payload, 1)
             mapping = np.full(count, -1, dtype=np.int64)
             mapping[live_idx] = new_offsets
+            if in_place:
+                self._defrag_in_place(stage, mapping, payload, new_size)
+                stats = {'original_size': original_size, 'new_size': self.bsonpath.stat().st_size,
+                         'reclaimed': original_size - self.bsonpath.stat().st_size, 'live_bytes': payload,
+                         'live_rows': int(len(live_idx)), 'logical_size': new_size}
+                Logger.log.info(
+                    f'Defragmented {self.relpath} BSON file in place: {original_size/1024/1024:.2f}MB -> '
+                    f'{new_size/1024/1024:.2f}MB logical (file {stats["new_size"]/1024/1024:.2f}MB)')
+                return stats
 
             def move_pointers(chunk, first):
                 moved = mapping[first:first + len(chunk)]
@@ -1834,10 +1896,6 @@ class Table:
             Exception: If the table is not schemaless or if defragmentation fails.
         """
         if self.type == 1:
-            from SharedData.TableRewrite import rewrite_supported
-            if not rewrite_supported():
-                Logger.log.warning(f'{self.relpath} BSON not compacted: the pair publication needs POSIX file APIs')
-                return {'original_size': 0, 'new_size': 0, 'reclaimed': 0}
             return self._defragment_bson_disk(chunk_size, acquire)
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')

{shareddata-6.85.11 → shareddata-6.85.14}/src/SharedData/TableDisk.py

@@ -13,6 +13,7 @@ from tqdm import tqdm
 import threading
 import psutil
 import shutil
+from SharedData.TableRewrite import remove_with_retry, rmtree_with_retry
 
 from SharedData.Logger import Logger
 from SharedData.IO.AWSS3 import S3DeleteFolder
@@ -178,6 +179,9 @@ class TableDisk(Table):
         # Header, size check and data come from one open file: trim replaces
         # data.bin, and np.memmap by path would map (and extend) the new file.
         with open(self.filepath, 'r+b') as stream:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                self._adopt_layout(stream)          # Windows: layout changes are published in place
             # memory map header — read the live header, never write it
             self.shf_hdr = np.memmap(stream, self.hdrdtype, 'r+', 0, (1,))
             self.hdr = self.shf_hdr[0]
@@ -199,6 +203,24 @@ class TableDisk(Table):
         self.records = SharedNumpy('DISK', self.shf_data)
         self.records.table = self
 
+    def _adopt_layout(self, stream):
+        """Follow a layout change published under the same inode (overwrite or reload with another schema,
+        in place): the header describes itself, so a process that still holds the old column list would map
+        the new bytes with the old record stride. Reparsed from the file, never from this object's copy."""
+        from SharedData.TableRewrite import RewriteError, read_layout
+        try:
+            header, records = read_layout(stream)
+        except RewriteError:
+            return                  # not a coherent image (yet): the caller's own checks report it
+        if header.dtype == self.hdrdtype and records == self.recdtype:
+            return
+        self.hdrdtype, self.recdtype = header.dtype, records
+        self.recnames, self.recformats = list(records.names), [str(records[n]) for n in records.names]
+        self.is_schemaless = 'bson_size' in self.recnames and 'bson_ptr' in self.recnames
+        self._record_bundle = None
+        self._index_bundle = None
+        self._reset_bson_mapping()
+
     ############### FREE ###############
     def free(self, acquire=True):        
         """
@@ -347,6 +369,9 @@ class TableDisk(Table):
             return self.records
         if change == 'tail' and not self.is_schemaless:
             return self._reload_tail()
+        from SharedData.TableRewrite import rewrite_supported
+        if not rewrite_supported():
+            return self._reload_image_in_place()
         require_rewrite_support()
         final = self.filepath
         old = os.stat(final)
@@ -401,6 +426,144 @@ class TableDisk(Table):
             raise Exception(errmsg)
         return self.records
 
+    def _apply_tail_in_place(self, header, tail_io, tailhdr, tail_at, capacity):
+        """Windows: the remote tail (already parsed up to its rows) is downloaded and checked against its
+        header and checksum into a private temporary file first; only then, in a STRUCT section under the
+        shared mutex, is the live file extended if the capacity grew, the tail rows are written through a
+        mapped view, rows beyond a shorter tail are zeroed, the header fields are published and the indexes
+        rebuilt. A failure before the STRUCT section leaves the live table untouched."""
+        import hashlib
+        import uuid
+        from SharedData.TableRewrite import RewriteError, extend_file, same_digest, sync_header, zero_mapped, copy_mapped
+        itemsize, tailsize = int(header['itemsize']), int(tailhdr['tailsize'])
+        final = self.filepath
+        temp = final.parent / ('data.bin.tail.' + uuid.uuid4().hex)
+        try:
+            tail_io.seek(tailhdr.dtype.itemsize)
+            digest, written = hashlib.md5(), 0
+            with open(temp, 'wb') as out:
+                while True:
+                    chunk = tail_io.read(CHUNK)
+                    if not chunk:
+                        break
+                    if written + len(chunk) > tailsize * itemsize:
+                        raise RewriteError('the remote tail of %s is longer than its header says' % self.relpath)
+                    digest.update(chunk)
+                    out.write(chunk)
+                    written += len(chunk)
+                out.flush()
+                os.fsync(out.fileno())
+            if written != tailsize * itemsize:
+                raise RewriteError('the remote tail of %s holds %d bytes, its header says %d' % (self.relpath, written, tailsize * itemsize))
+            if not same_digest(digest.digest(), tailhdr['md5hash']):
+                raise RewriteError('the remote tail of %s does not match its checksum' % self.relpath)
+            with structure(self):
+                hdrsize = int(self.hdr.dtype.itemsize)
+                old_end = hdrsize + int(self.hdr['count']) * itemsize
+                new_end = tail_at + written
+                extend_file(final, hdrsize + capacity * itemsize)
+                self.shf_data.flush()
+                copy_mapped(final, temp, written, dst_offset=tail_at)
+                zero_mapped(final, new_end, old_end)            # a shorter tail than the local one leaves no old rows behind
+                for field in ('md5hashtail', 'tailsize', 'count', 'mtimetail', 'recordssize', 'isidxcreated',
+                              'isloaded', 'isidxsorted', 'minchgid'):
+                    self.hdr[field] = header[field]
+                sync_header(self)
+                self.size = capacity
+                self._attach_replacement(final, rebuild_files=False)
+            return self.records
+        finally:
+            try:
+                temp.unlink()
+            except OSError:
+                pass
+
+    def _reload_image_in_place(self):
+        """Windows: the whole remote image is downloaded beside the live files (the live BSON path is
+        redirected too, so nothing live is written while preparing), then, inside the STRUCT section the
+        caller holds, copied over the live files through mapped views: the file is extended when the remote
+        capacity is larger, committed rows left over from a longer local image are zeroed, the BSON file is
+        copied and shrunk when no other process maps it. Index files are kept (rebuilt, oversized is fine)."""
+        import gc
+        import uuid
+        from SharedData.TableRewrite import (RewriteError, copy_mapped, extend_file, read_layout, shrink_file,
+                                             zero_mapped)
+        final, bson_final = self.filepath, self.bsonpath
+        for leftover in list(final.parent.glob('data.bin.reload.*')) + list(final.parent.glob('bson.bin.reload.*')):
+            try:
+                leftover.unlink()
+            except OSError:
+                pass
+        token = uuid.uuid4().hex
+        stage = final.parent / ('data.bin.reload.' + token)
+        bson_stage = final.parent / ('bson.bin.reload.' + token)
+        old_end = int(self.hdr.dtype.itemsize) + int(self.hdr['count']) * int(self.recdtype.itemsize)
+        old_bson = bson_final.stat().st_size if bson_final.exists() else 0
+        old_layout = (self.hdrdtype, self.recdtype, getattr(self, 'recnames', None), getattr(self, 'recformats', None),
+                      self.is_schemaless)
+        saved = (self.hdr, self.size, getattr(self, 'orig_hdr', None))
+        self.filepath, self.bsonpath, self.hdr = stage, bson_stage, None
+        self._bootstrap_record_download = True
+        try:
+            self._download_empty_image()
+            if self.hdr is None or not stage.is_file():
+                raise Exception('the remote copy of %s disappeared during the reload' % self.relpath)
+            self.free(acquire=False)                                 # the maps of the stage files
+            gc.collect()
+            with open(stage, 'rb') as stream:
+                downloaded, records = read_layout(stream)
+            hdrsize = int(downloaded.dtype.itemsize)
+            commit = hdrsize + int(downloaded['count']) * int(records.itemsize)
+            capacity_bytes = hdrsize + int(downloaded['recordssize']) * int(records.itemsize)
+            new_hdrdtype, new_recdtype, new_size = self.hdrdtype, self.recdtype, self.size
+            new_schemaless = self.is_schemaless
+            have_bson = bson_stage.exists()
+            self.filepath, self.bsonpath = final, bson_final
+            self._reset_bson_mapping(reopen=False)
+            extend_file(final, capacity_bytes)
+            copy_mapped(final, stage, commit)
+            zero_mapped(final, commit, old_end)
+            if have_bson:
+                bson_size = bson_stage.stat().st_size
+                if not bson_final.exists():
+                    bson_final.touch()
+                extend_file(bson_final, bson_size)
+                copy_mapped(bson_final, bson_stage, bson_size)
+                physical = bson_final.stat().st_size
+                if not shrink_file(bson_final, bson_size) and physical > bson_size:
+                    zero_mapped(bson_final, bson_size, physical)
+        except BaseException:
+            self.filepath, self.bsonpath = final, bson_final
+            self.hdr, self.size, self.orig_hdr = saved
+            self.hdrdtype, self.recdtype = old_layout[0], old_layout[1]
+            if old_layout[2] is not None:
+                self.recnames, self.recformats = old_layout[2], old_layout[3]
+            self.is_schemaless = old_layout[4]
+            raise
+        finally:
+            self._bootstrap_record_download = False
+            for leftover in (stage, bson_stage):
+                try:
+                    leftover.unlink()
+                except OSError:
+                    pass
+        try:
+            self.free(acquire=False)
+            self.malloc()
+            if self.is_schemaless:
+                self.init_bson_mmap(from_download=have_bson)
+            if self.hasindex:
+                self.hdr['isidxcreated'] = 0
+                self.index.initialize()
+            self.shareddata.data[self.relpath] = self
+            self.hdr['isloaded'] = 1
+            self.mutex['isloaded'] = 1
+        except Exception as e:
+            errmsg = f"TableDisk.reload() {self.relpath}: {e}"
+            Logger.log.error(errmsg)
+            raise Exception(errmsg)
+        return self.records
+
     def _attach_replacement(self, final, rebuild_files):
         """After os.replace: make the directory entry durable, then map the new image and rebuild its
         indexes. Whatever fails, this object ends mapped to the image that is now installed (flagged
@@ -454,7 +617,8 @@ class TableDisk(Table):
         from SharedData.TableRewrite import (RewriteError, copy_metadata, pwrite_all, require_rewrite_support,
                                              same_digest, sync_header)
         import hashlib
-        require_rewrite_support()
+        from SharedData.TableRewrite import rewrite_supported
+        in_place = not rewrite_supported()
         acquired = not self._owns_lock()
         if acquired:
             self.acquire()
@@ -489,6 +653,8 @@ class TableDisk(Table):
             header['isidxsorted'] = 1                                 # the uploader sorts before it writes (SharedNumpy.write)
             pending = int(header['minchgid'])                         # a head row edited here and not uploaded yet stays marked for upload;
             header['minchgid'] = pending if pending <= int(header['headsize']) else int(header['count']) + 1   # otherwise as a write leaves it
+            if in_place:
+                return self._apply_tail_in_place(header, tail_io, tailhdr, tail_at, capacity)
             for leftover in final.parent.glob('data.bin.reload.*'):   # the lock is held: a stage left by a dead reload
                 leftover.unlink()
             stage = final.parent / ('data.bin.reload.' + uuid.uuid4().hex)
@@ -574,10 +740,10 @@ class TableDisk(Table):
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                 # if folder is empty remove it
                 if not any(localpath.iterdir()):
-                    shutil.rmtree(localpath)            
+                    rmtree_with_retry(localpath)
 
             success = True