shareddata 6.85.10__tar.gz → 6.85.12__tar.gz

{shareddata-6.85.10/src/shareddata.egg-info → shareddata-6.85.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.10
+Version: 6.85.12
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.10 → shareddata-6.85.12}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.10',
+    version='6.85.12',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/CollectionMongoDB.py

@@ -321,7 +321,7 @@ class CollectionMongoDB:
         # Insert all prepared documents into the collection        
         result = []
         if documents_to_insert:
-            result = self._collection.insert_many(documents_to_insert)
+            result = self._collection.insert_many(documents_to_insert, ordered=False)
         
         return result        
     

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/SharedData.py

@@ -6,6 +6,7 @@ import json
 import warnings
 import time
 import shutil
+from SharedData.TableRewrite import remove_with_retry, rmtree_with_retry
 from datetime import datetime, timezone
 
 from multiprocessing import shared_memory
@@ -989,7 +990,7 @@ class SharedData:
                 fpath = Path(os.environ['DATABASE_FOLDER'])
                 fpath = fpath/('shm/'+shm_name.replace('\\', '/')+'.csv')
                 if fpath.is_file():
-                    os.remove(fpath)
+                    remove_with_retry(fpath)
             except:
                 pass
 
@@ -1685,7 +1686,7 @@ class SharedData:
                     except:
                         try:
                             if fpath.is_file():
-                                os.remove(fpath)
+                                remove_with_retry(fpath)
                         except:
                             pass
         shm_names = shm_names.sort_index()
@@ -1845,11 +1846,11 @@ class SharedData:
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                 # if folder is empty remove it
                 if localpath.exists():
                     if not any(localpath.iterdir()):
-                        shutil.rmtree(localpath)
+                        rmtree_with_retry(localpath)
             else:           
                 database_folder = os.environ['DATABASE_FOLDER']     
                 # select the first disk with less percent_used
@@ -1865,10 +1866,10 @@ class SharedData:
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                     # if folder is empty remove it
                     if not any(localpath.iterdir()):
-                        shutil.rmtree(localpath)
+                        rmtree_with_retry(localpath)
                 
             from SharedData.services import s3_enabled
             if not localonly and s3_enabled():
@@ -1911,12 +1912,12 @@ class SharedData:
                     del self.data[path]                
                 localpath = Path(os.environ['DATABASE_FOLDER'])/Path(path)
                 if localpath.exists():
-                    shutil.rmtree(localpath)
+                    rmtree_with_retry(localpath)
                 files_del = ['timeseries_head.bin','timeseries_tail.bin']
                 for file in files_del:
                     delpath = Path(localpath.parent/file)
                     if delpath.exists():
-                        os.remove(delpath)
+                        remove_with_retry(delpath)
                 from SharedData.services import s3_enabled
                 if not localonly and s3_enabled():
                     S3DeleteTimeseries(path.replace('/timeseries',''))
@@ -1928,7 +1929,7 @@ class SharedData:
                 fpath, shm_name = tstag.get_path()
                 del self.data[path].tags[tag]
                 del ts
-                os.remove(fpath)
+                remove_with_retry(fpath)
                 return True
             
         except Exception as e:

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/SharedNumpy.py

@@ -297,8 +297,9 @@ class SharedNumpy(np.ndarray):
             force_write (bool): If True, forces the write operation even if not necessary. Defaults to False.
         """
         # Saving remains available without unsupported record replacement.
-        if (self.table.type != 1 or sys.platform == 'linux') and (
-                self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):
+        from SharedData.TableRewrite import rewrite_supported
+        if (self.table.type != 1 or sys.platform == 'linux' or not rewrite_supported()) and (
+                self.table.hdr['hasindex'] == 1 and self.table.hdr['isidxsorted'] == 0):      # Windows sorts in place
             self.sort_index()
         self.table.write(force_write, strict)
         

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/Table.py

@@ -687,8 +687,29 @@ class Table:
         records, so they are removed and rebuilt from the new empty header.
         """
         from SharedData.TableRewrite import (RewriteError, fsync_directory,
-                                             require_rewrite_support)
-        require_rewrite_support()
+                                             rewrite_supported)
+        if not rewrite_supported():
+            # Windows cannot rename over or unlink a file any process maps. The shared mutex is held in STRUCT
+            # mode, so the image is re-initialised in place: the old committed rows and the header are zeroed
+            # through a mapped view (coherent with every other process's view), the file is only ever extended,
+            # and the index / BSON files stay as they are (the index is rebuilt by the caller, and the new
+            # header resets next_bson_ptr, so no old document is reachable).
+            from SharedData.TableRewrite import RewriteError, extend_file, read_layout
+            final = self.filepath
+            try:
+                with open(final, 'rb') as stream:
+                    old_header, old_records = read_layout(stream)
+                clear = old_header.dtype.itemsize + int(old_header['count']) * old_records.itemsize
+            except RewriteError:
+                clear = os.path.getsize(final)          # not a coherent image: clear everything it holds
+            extend_file(final, int(self.hdrdtype.itemsize + self.recdtype.itemsize * int(self.size)))
+            clear = max(clear, int(self.hdrdtype.itemsize))
+            view = np.memmap(final, np.uint8, 'r+', 0, (clear,))
+            view[:] = 0
+            view.flush()
+            del view
+            self.malloc(create=True)
+            return
         final = self.filepath
         old = os.stat(final)
         # The writer lock is held: any earlier stage is from a build that died.
@@ -1385,7 +1406,7 @@ class Table:
         return h1[:l] == h2[:l]
 
     ############### BSON MANAGEMENT ###############
-    def _reset_bson_mapping(self):
+    def _reset_bson_mapping(self, reopen=True):
         """Drop the BSON mapping and reopen the file by path: after a paired publication the old
         mapping belongs to an unlinked inode and its offsets no longer match the records."""
         if self.bson_mmap is not None:
@@ -1399,7 +1420,7 @@ class Table:
             self.bson_file_handle = None
         self.curr_bson_size = 0
         self.prev_bson_ptr = 0
-        if self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
+        if reopen and self.is_schemaless and self.bsonpath.exists() and self.bsonpath.stat().st_size > 0:
             self.init_bson_mmap()
 
     def init_bson_mmap(self, from_download=False):
@@ -1668,6 +1689,50 @@ class Table:
             return None
         return self.defragment_bson(acquire=False)
 
+    def _defrag_in_place(self, stage, mapping, payload, new_size):
+        """Windows: the compacted BSON `stage` is copied over the start of the live file and every moved
+        bson_ptr is rewritten in place, as one STRUCT section under the shared mutex (nothing else touches the
+        table), then the file is shrunk when no process maps it. Not crash-safe (a crash between the BSON
+        copy and the pointer update leaves STRUCT/DIRTY and pointers into a rewritten file); the staged
+        publication that covers that needs Linux."""
+        from SharedData.TableProtocol import structure
+        from SharedData.TableRewrite import shrink_file, sync_header
+        count = int(self.hdr['count'])
+        with structure(self):
+            if self.bson_mmap is not None:
+                self.bson_mmap.flush()
+            data = np.fromfile(stage, dtype=np.uint8)          # the stage is the compacted payload (+1 pad byte)
+            view = np.memmap(self.bsonpath, np.uint8, 'r+', 0, (len(data),))
+            view[:] = data
+            view.flush()
+            del view
+            rows = self.shf_data[:count]                       # the file's own mapping, never a caller-visible copy
+            selected = mapping >= 0
+            if selected.any():
+                column = rows['bson_ptr']
+                if column.dtype.kind == 'V' and column.dtype.itemsize == 8:
+                    column = column.view(np.int64)
+                column[selected] = mapping[selected]
+            self.shf_data.flush()
+            self.hdr['next_bson_ptr'] = payload
+            self.hdr['mtime'] = max(datetime.now().timestamp(), float(self.hdr['mtime']) + 0.001)
+            sync_header(self)
+            self.attach()                                      # records/header of this object: the mapping of the file, as after a publication
+            self._reset_bson_mapping(reopen=False)             # drop this process's map, then try to give the space back
+            physical = self.bsonpath.stat().st_size
+            if not shrink_file(self.bsonpath, new_size) and physical > new_size:
+                # another process still maps the file: it stays as large, but the stale documents behind the
+                # new end are zeroed so the upload (which compresses the file to its end) stays small
+                tail = np.memmap(self.bsonpath, np.uint8, 'r+', new_size, (physical - new_size,))
+                tail[:] = 0
+                tail.flush()
+                del tail
+            self._reset_bson_mapping()
+        try:
+            Path(stage).unlink()
+        except OSError:
+            pass
+
     def _defragment_bson_disk(self, chunk_size, acquire):
         """Compact bson.bin of a disk table and move every bson_ptr, as ONE crash-safe publication.
 
@@ -1681,7 +1746,8 @@ class Table:
         from SharedData.TableRewrite import (RewriteError, copy_metadata, file_identity, fsync_directory,
                                              pending_path, publish_rewrite, require_rewrite_support, write_all)
         from SharedData.TableProtocol import structure
-        require_rewrite_support()
+        from SharedData.TableRewrite import rewrite_supported
+        in_place = not rewrite_supported()
         if not self.is_schemaless:
             raise Exception(f'Cannot defragment non-schemaless table {self.relpath}')
         if acquire:
@@ -1695,8 +1761,8 @@ class Table:
             if self.bson_mmap is not None:
                 self.bson_mmap.flush()
             if self.bsonpath.exists():
-                with open(self.bsonpath, 'rb') as baseline:
-                    os.fsync(baseline.fileno())
+                from SharedData.TableRewrite import sync_file
+                sync_file(self.bsonpath)                    # r+b on Windows: fsync needs a writable handle
             count = int(self.hdr['count'])
             pointers, sizes = self._bson_columns(count) if count else (np.zeros(0, np.int64), np.zeros(0, np.int64))
             live_idx = np.flatnonzero(sizes > 0)
@@ -1722,9 +1788,10 @@ class Table:
             token = uuid.uuid4().hex
             stage = self.bsonpath.parent / ('bson.bin.stage.' + token)
             digest = hashlib.sha256()
-            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+            fd = os.open(stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0), 0o600)
             try:
-                copy_metadata(self.bsonpath, fd)
+                if not in_place:
+                    copy_metadata(self.bsonpath, fd)
                 if len(live_idx):
                     with open(self.bsonpath, 'rb') as source:
                         view = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
@@ -1751,6 +1818,15 @@ class Table:
             new_size = max(payload, 1)
             mapping = np.full(count, -1, dtype=np.int64)
             mapping[live_idx] = new_offsets
+            if in_place:
+                self._defrag_in_place(stage, mapping, payload, new_size)
+                stats = {'original_size': original_size, 'new_size': self.bsonpath.stat().st_size,
+                         'reclaimed': original_size - self.bsonpath.stat().st_size, 'live_bytes': payload,
+                         'live_rows': int(len(live_idx)), 'logical_size': new_size}
+                Logger.log.info(
+                    f'Defragmented {self.relpath} BSON file in place: {original_size/1024/1024:.2f}MB -> '
+                    f'{new_size/1024/1024:.2f}MB logical (file {stats["new_size"]/1024/1024:.2f}MB)')
+                return stats
 
             def move_pointers(chunk, first):
                 moved = mapping[first:first + len(chunk)]

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/TableDisk.py

@@ -13,6 +13,7 @@ from tqdm import tqdm
 import threading
 import psutil
 import shutil
+from SharedData.TableRewrite import remove_with_retry, rmtree_with_retry
 
 from SharedData.Logger import Logger
 from SharedData.IO.AWSS3 import S3DeleteFolder
@@ -178,6 +179,9 @@ class TableDisk(Table):
         # Header, size check and data come from one open file: trim replaces
         # data.bin, and np.memmap by path would map (and extend) the new file.
         with open(self.filepath, 'r+b') as stream:
+            from SharedData.TableRewrite import rewrite_supported
+            if not rewrite_supported():
+                self._adopt_layout(stream)          # Windows: layout changes are published in place
             # memory map header — read the live header, never write it
             self.shf_hdr = np.memmap(stream, self.hdrdtype, 'r+', 0, (1,))
             self.hdr = self.shf_hdr[0]
@@ -199,6 +203,24 @@ class TableDisk(Table):
         self.records = SharedNumpy('DISK', self.shf_data)
         self.records.table = self
 
+    def _adopt_layout(self, stream):
+        """Follow a layout change published under the same inode (overwrite or reload with another schema,
+        in place): the header describes itself, so a process that still holds the old column list would map
+        the new bytes with the old record stride. Reparsed from the file, never from this object's copy."""
+        from SharedData.TableRewrite import RewriteError, read_layout
+        try:
+            header, records = read_layout(stream)
+        except RewriteError:
+            return                  # not a coherent image (yet): the caller's own checks report it
+        if header.dtype == self.hdrdtype and records == self.recdtype:
+            return
+        self.hdrdtype, self.recdtype = header.dtype, records
+        self.recnames, self.recformats = list(records.names), [str(records[n]) for n in records.names]
+        self.is_schemaless = 'bson_size' in self.recnames and 'bson_ptr' in self.recnames
+        self._record_bundle = None
+        self._index_bundle = None
+        self._reset_bson_mapping()
+
     ############### FREE ###############
     def free(self, acquire=True):        
         """
@@ -300,7 +322,12 @@ class TableDisk(Table):
         """
         if self.hdr['recordssize'] <= self.hdr['count']:
             return self
-        from SharedData.TableRewrite import publish_rewrite
+        from SharedData.TableRewrite import publish_rewrite, rewrite_supported
+        if not rewrite_supported():
+            # Shrinking needs a second file replacing the mapped one (Linux). Trim only returns unused space,
+            # so the table is left as it is rather than failing.
+            Logger.log.warning('%s not trimmed: shrinking a disk table needs POSIX file APIs' % self.relpath)
+            return self
         return publish_rewrite(self, None, 'trim', capacity=max(2, int(self.hdr['count'])))
 
     ############### RELOAD ###############
@@ -342,6 +369,9 @@ class TableDisk(Table):
             return self.records
         if change == 'tail' and not self.is_schemaless:
             return self._reload_tail()
+        from SharedData.TableRewrite import rewrite_supported
+        if not rewrite_supported():
+            return self._reload_image_in_place()
         require_rewrite_support()
         final = self.filepath
         old = os.stat(final)
@@ -396,6 +426,144 @@ class TableDisk(Table):
             raise Exception(errmsg)
         return self.records
 
+    def _apply_tail_in_place(self, header, tail_io, tailhdr, tail_at, capacity):
+        """Windows: the remote tail (already parsed up to its rows) is downloaded and checked against its
+        header and checksum into a private temporary file first; only then, in a STRUCT section under the
+        shared mutex, is the live file extended if the capacity grew, the tail rows are written through a
+        mapped view, rows beyond a shorter tail are zeroed, the header fields are published and the indexes
+        rebuilt. A failure before the STRUCT section leaves the live table untouched."""
+        import hashlib
+        import uuid
+        from SharedData.TableRewrite import RewriteError, extend_file, same_digest, sync_header, zero_mapped, copy_mapped
+        itemsize, tailsize = int(header['itemsize']), int(tailhdr['tailsize'])
+        final = self.filepath
+        temp = final.parent / ('data.bin.tail.' + uuid.uuid4().hex)
+        try:
+            tail_io.seek(tailhdr.dtype.itemsize)
+            digest, written = hashlib.md5(), 0
+            with open(temp, 'wb') as out:
+                while True:
+                    chunk = tail_io.read(CHUNK)
+                    if not chunk:
+                        break
+                    if written + len(chunk) > tailsize * itemsize:
+                        raise RewriteError('the remote tail of %s is longer than its header says' % self.relpath)
+                    digest.update(chunk)
+                    out.write(chunk)
+                    written += len(chunk)
+                out.flush()
+                os.fsync(out.fileno())
+            if written != tailsize * itemsize:
+                raise RewriteError('the remote tail of %s holds %d bytes, its header says %d' % (self.relpath, written, tailsize * itemsize))
+            if not same_digest(digest.digest(), tailhdr['md5hash']):
+                raise RewriteError('the remote tail of %s does not match its checksum' % self.relpath)
+            with structure(self):
+                hdrsize = int(self.hdr.dtype.itemsize)
+                old_end = hdrsize + int(self.hdr['count']) * itemsize
+                new_end = tail_at + written
+                extend_file(final, hdrsize + capacity * itemsize)
+                self.shf_data.flush()
+                copy_mapped(final, temp, written, dst_offset=tail_at)
+                zero_mapped(final, new_end, old_end)            # a shorter tail than the local one leaves no old rows behind
+                for field in ('md5hashtail', 'tailsize', 'count', 'mtimetail', 'recordssize', 'isidxcreated',
+                              'isloaded', 'isidxsorted', 'minchgid'):
+                    self.hdr[field] = header[field]
+                sync_header(self)
+                self.size = capacity
+                self._attach_replacement(final, rebuild_files=False)
+            return self.records
+        finally:
+            try:
+                temp.unlink()
+            except OSError:
+                pass
+
+    def _reload_image_in_place(self):
+        """Windows: the whole remote image is downloaded beside the live files (the live BSON path is
+        redirected too, so nothing live is written while preparing), then, inside the STRUCT section the
+        caller holds, copied over the live files through mapped views: the file is extended when the remote
+        capacity is larger, committed rows left over from a longer local image are zeroed, the BSON file is
+        copied and shrunk when no other process maps it. Index files are kept (rebuilt, oversized is fine)."""
+        import gc
+        import uuid
+        from SharedData.TableRewrite import (RewriteError, copy_mapped, extend_file, read_layout, shrink_file,
+                                             zero_mapped)
+        final, bson_final = self.filepath, self.bsonpath
+        for leftover in list(final.parent.glob('data.bin.reload.*')) + list(final.parent.glob('bson.bin.reload.*')):
+            try:
+                leftover.unlink()
+            except OSError:
+                pass
+        token = uuid.uuid4().hex
+        stage = final.parent / ('data.bin.reload.' + token)
+        bson_stage = final.parent / ('bson.bin.reload.' + token)
+        old_end = int(self.hdr.dtype.itemsize) + int(self.hdr['count']) * int(self.recdtype.itemsize)
+        old_bson = bson_final.stat().st_size if bson_final.exists() else 0
+        old_layout = (self.hdrdtype, self.recdtype, getattr(self, 'recnames', None), getattr(self, 'recformats', None),
+                      self.is_schemaless)
+        saved = (self.hdr, self.size, getattr(self, 'orig_hdr', None))
+        self.filepath, self.bsonpath, self.hdr = stage, bson_stage, None
+        self._bootstrap_record_download = True
+        try:
+            self._download_empty_image()
+            if self.hdr is None or not stage.is_file():
+                raise Exception('the remote copy of %s disappeared during the reload' % self.relpath)
+            self.free(acquire=False)                                 # the maps of the stage files
+            gc.collect()
+            with open(stage, 'rb') as stream:
+                downloaded, records = read_layout(stream)
+            hdrsize = int(downloaded.dtype.itemsize)
+            commit = hdrsize + int(downloaded['count']) * int(records.itemsize)
+            capacity_bytes = hdrsize + int(downloaded['recordssize']) * int(records.itemsize)
+            new_hdrdtype, new_recdtype, new_size = self.hdrdtype, self.recdtype, self.size
+            new_schemaless = self.is_schemaless
+            have_bson = bson_stage.exists()
+            self.filepath, self.bsonpath = final, bson_final
+            self._reset_bson_mapping(reopen=False)
+            extend_file(final, capacity_bytes)
+            copy_mapped(final, stage, commit)
+            zero_mapped(final, commit, old_end)
+            if have_bson:
+                bson_size = bson_stage.stat().st_size
+                if not bson_final.exists():
+                    bson_final.touch()
+                extend_file(bson_final, bson_size)
+                copy_mapped(bson_final, bson_stage, bson_size)
+                physical = bson_final.stat().st_size
+                if not shrink_file(bson_final, bson_size) and physical > bson_size:
+                    zero_mapped(bson_final, bson_size, physical)
+        except BaseException:
+            self.filepath, self.bsonpath = final, bson_final
+            self.hdr, self.size, self.orig_hdr = saved
+            self.hdrdtype, self.recdtype = old_layout[0], old_layout[1]
+            if old_layout[2] is not None:
+                self.recnames, self.recformats = old_layout[2], old_layout[3]
+            self.is_schemaless = old_layout[4]
+            raise
+        finally:
+            self._bootstrap_record_download = False
+            for leftover in (stage, bson_stage):
+                try:
+                    leftover.unlink()
+                except OSError:
+                    pass
+        try:
+            self.free(acquire=False)
+            self.malloc()
+            if self.is_schemaless:
+                self.init_bson_mmap(from_download=have_bson)
+            if self.hasindex:
+                self.hdr['isidxcreated'] = 0
+                self.index.initialize()
+            self.shareddata.data[self.relpath] = self
+            self.hdr['isloaded'] = 1
+            self.mutex['isloaded'] = 1
+        except Exception as e:
+            errmsg = f"TableDisk.reload() {self.relpath}: {e}"
+            Logger.log.error(errmsg)
+            raise Exception(errmsg)
+        return self.records
+
     def _attach_replacement(self, final, rebuild_files):
         """After os.replace: make the directory entry durable, then map the new image and rebuild its
         indexes. Whatever fails, this object ends mapped to the image that is now installed (flagged
@@ -449,7 +617,8 @@ class TableDisk(Table):
         from SharedData.TableRewrite import (RewriteError, copy_metadata, pwrite_all, require_rewrite_support,
                                              same_digest, sync_header)
         import hashlib
-        require_rewrite_support()
+        from SharedData.TableRewrite import rewrite_supported
+        in_place = not rewrite_supported()
         acquired = not self._owns_lock()
         if acquired:
             self.acquire()
@@ -484,6 +653,8 @@ class TableDisk(Table):
             header['isidxsorted'] = 1                                 # the uploader sorts before it writes (SharedNumpy.write)
             pending = int(header['minchgid'])                         # a head row edited here and not uploaded yet stays marked for upload;
             header['minchgid'] = pending if pending <= int(header['headsize']) else int(header['count']) + 1   # otherwise as a write leaves it
+            if in_place:
+                return self._apply_tail_in_place(header, tail_io, tailhdr, tail_at, capacity)
             for leftover in final.parent.glob('data.bin.reload.*'):   # the lock is held: a stage left by a dead reload
                 leftover.unlink()
             stage = final.parent / ('data.bin.reload.' + uuid.uuid4().hex)
@@ -569,10 +740,10 @@ class TableDisk(Table):
                     for file in delfiles:
                         delpath = Path(localpath/file)
                         if delpath.exists():
-                            os.remove(delpath)
+                            remove_with_retry(delpath)
                 # if folder is empty remove it
                 if not any(localpath.iterdir()):
-                    shutil.rmtree(localpath)            
+                    rmtree_with_retry(localpath)
 
             success = True
             

{shareddata-6.85.10 → shareddata-6.85.12}/src/SharedData/TableIndex.py

@@ -257,7 +257,7 @@ class TableIndex:
             resetfile = False
             if (not os.path.exists(self.pkeypath)):
                 resetfile = True
-            elif os.path.getsize(self.pkeypath) != keysize_bytes:
+            elif not self._index_file_fits(self.pkeypath, keysize_bytes):
                 resetfile = True            
 
             if resetfile:
@@ -307,7 +307,7 @@ class TableIndex:
             resetfile = False
             if (not os.path.exists(self.dateidxpath)):
                 resetfile = True                
-            elif os.path.getsize(self.dateidxpath) != size_bytes:
+            elif not self._index_file_fits(self.dateidxpath, size_bytes):
                 resetfile = True
 
             if resetfile:
@@ -365,7 +365,7 @@ class TableIndex:
             resetfile = False
             if (not os.path.exists(self.symbolidxpath)):
                 resetfile = True                
-            elif os.path.getsize(self.symbolidxpath) != size_bytes:
+            elif not self._index_file_fits(self.symbolidxpath, size_bytes):
                 resetfile = True
 
             if resetfile:
@@ -425,7 +425,7 @@ class TableIndex:
             resetfile = False
             if (not os.path.exists(self.portidxpath)):
                 resetfile = True                
-            elif os.path.getsize(self.portidxpath) != size_bytes:
+            elif not self._index_file_fits(self.portidxpath, size_bytes):
                 resetfile = True
             if resetfile:
                 self.create_file(self.portidxpath,size_bytes)
@@ -618,14 +618,32 @@ class TableIndex:
             Logger.log.error('Error sorting index!\n%s' % (e))
             raise
 
+    @staticmethod
+    def _inplace_files():
+        """True where an index file cannot be replaced while mapped (Windows): it is extended instead."""
+        from SharedData.TableRewrite import rewrite_supported
+        return not rewrite_supported()
+
+    def _stream_fits(self, stream, size_bytes):
+        size = os.fstat(stream.fileno()).st_size
+        return size == size_bytes or (size > size_bytes and self._inplace_files())
+
+    def _index_file_fits(self, path, size_bytes):
+        """The file can back an index of `size_bytes`: exactly that size, or (in-place platforms) larger.
+
+        Maps are always created with an explicit shape, so an oversized file is harmless; shrinking one
+        would need every process to drop its mapping first, which Windows enforces and cannot be asked for."""
+        size = os.path.getsize(path)
+        return size == size_bytes or (size > size_bytes and self._inplace_files())
+
     def create_file(self,fpath,size):
         """
         Creates a file at the specified path with the given size.
         
-        This method opens a file in binary write mode, seeks to the byte position size-1,
-        and writes a single null byte to allocate the file space. On POSIX systems, it uses
-        os.posix_fallocate to efficiently allocate disk space. On Windows systems, the allocation
-        step is currently not implemented.
+        Linux: a complete replacement file is built beside the target, synced, and installed with one
+        os.replace (os.posix_fallocate reserves the space). Windows cannot rename over a mapped file, so an
+        existing file is only ever extended in place (extension is allowed with live mappings; the arrays
+        are re-initialised by the caller).
         
         Parameters:
             fpath (str): The file path where the file will be created.
@@ -634,6 +652,19 @@ class TableIndex:
         Raises:
             OSError: If file operations or allocation fail.
         """
+        from SharedData.TableRewrite import checkpoint, extend_file, fsync_directory, sync_header
+        if self._inplace_files():
+            if os.path.exists(fpath):
+                extend_file(fpath, size)
+            else:
+                with open(fpath, 'wb') as f:
+                    f.seek(size-1)
+                    f.write(b'\x00')
+                    f.flush()
+                    os.fsync(f.fileno())
+            self.table.hdr['isidxcreated'] = 0
+            sync_header(self.table)
+            return
         temporary = str(fpath) + '.tmp.' + uuid.uuid4().hex
         with open(temporary, 'wb') as f:
             # Seek to the end of the file
@@ -642,11 +673,8 @@ class TableIndex:
             f.write(b'\x00')
             if os.name == 'posix':
                 os.posix_fallocate(f.fileno(), 0, size)
-            elif os.name == 'nt':
-                pass # TODO: implement windows file allocation
             f.flush()
             os.fsync(f.fileno())
-        from SharedData.TableRewrite import checkpoint, fsync_directory, sync_header
         self.table.hdr['isidxcreated'] = 0
         sync_header(self.table)
         checkpoint('before_index_replace_' + os.path.basename(fpath))
@@ -689,7 +717,15 @@ class TableIndex:
         # smaller file to the size checked: a lock-free reader resized the index.
         # Infer keysize from actual file size
         with open(self.pkeypath, 'r+b') as stream:
-            keysize = os.fstat(stream.fileno()).st_size // 8  # int64 = 8 bytes
+            filesize = os.fstat(stream.fileno()).st_size
+            if self._inplace_files():
+                # the file may be larger than the table needs (it is never shrunk): the hash modulus is
+                # the table's, not the file's
+                keysize = int(self.table.size * 3)
+                if filesize < keysize * 8:
+                    raise Exception(f'Primary key index file is smaller than expected: {self.pkeypath}')
+            else:
+                keysize = filesize // 8  # int64 = 8 bytes
             self.pkey = np.memmap(stream, np.int64, 'r+', 0, (keysize,))
         
         # Remap date indices if present
@@ -707,7 +743,7 @@ class TableIndex:
             
             # Check if file size matches
             with open(self.dateidxpath, 'r+b') as stream:
-                if os.fstat(stream.fileno()).st_size == expected_size:
+                if self._stream_fits(stream, expected_size):
                     self.datelastidx = np.memmap(stream, np.int64, 'r+', 0, (dateidxsize,))
                     self.dateprevidx = np.memmap(stream, np.int64, 'r+', hashtblsize_bytes, (listsize,))
                 else:
@@ -726,7 +762,7 @@ class TableIndex:
             
             # Check if file size matches
             with open(self.symbolidxpath, 'r+b') as stream:
-                if os.fstat(stream.fileno()).st_size == expected_size:
+                if self._stream_fits(stream, expected_size):
                     self.symbollastidx = np.memmap(stream, np.int64, 'r+', 0, (hashtblsize,))
                     self.symbolprevidx = np.memmap(stream, np.int64, 'r+', hashtblsize_bytes, (listsize,))
                 else:
@@ -745,7 +781,7 @@ class TableIndex:
             
             # Check if file size matches
             with open(self.portidxpath, 'r+b') as stream:
-                if os.fstat(stream.fileno()).st_size == expected_size:
+                if self._stream_fits(stream, expected_size):
                     self.portlastidx = np.memmap(stream, np.int64, 'r+', 0, (hashtblsize,))
                     self.portprevidx = np.memmap(stream, np.int64, 'r+', hashtblsize_bytes, (listsize,))
                 else: