arrayfile 0.0.1__tar.gz → 0.1.0__tar.gz

{arrayfile-0.0.1 → arrayfile-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arrayfile
-Version: 0.0.1
+Version: 0.1.0
 Summary: Arrays backed by disk
 Keywords: logging,terminal,scrollback,indexing
 Author-email: Gareth Davidson <gaz@bitplane.net>
@@ -27,14 +27,14 @@ Requires-Dist: ruff ; extra == "dev"
 Requires-Dist: pydoc-markdown ; extra == "dev"
 Provides-Extra: dev
 
-# arrayfile
+# 📃 arrayfile
 
 A file-backed numeric array using struct.pack. Does not support inserts or
 slicing.
 
 Smaller than relying on numpy though.
 
-## Installation
+## ▶️ Installation
 
 ```bash
 pip install arrayfile
@@ -42,7 +42,7 @@ pip install arrayfile
 
 ## Usage
 
-### Temporary Array
+### 🗑 Temporary Array
 
 This creates an array in your temp dir:
 
@@ -60,7 +60,7 @@ print(f"Values: {[arr[i] for i in range(len(arr))]}")
 arr.close()  # Clean up resources
 ```
 
-### Persistent Array
+### 💽 Persistent Array
 
 You can use the same file, if you want to persist your data across sessions:
 
@@ -84,7 +84,7 @@ arr.append(2000)
 arr.close()
 ```
 
-## Context manager
+## 🔧 Context manager
 
 It has a finalizer in case you forget to call `close()`, but if you like to keep
 your code tidy, you can use a context manager, like so:
@@ -101,4 +101,19 @@ with Array('d', 'measurements.array', 'w+b') as arr:
         print(f"  {i}: {value:.15f}")
 ```
 
+## ⚖️ License
+
+WTFPL with one extra clause:
+
+1. Don't blame me.
+
+Do what you like, but you're on your own.
+
+## 🔗 Links
+
+* [🏠 home](https://bitplane.net/dev/python/arrayfile)
+* [📖 pydoc](https://bitplane.net/dev/python/arrayfile/pydoc)
+* [🐍 pypi](https://pypi.org/project/arrayfile)
+* [🐱 github](https://github.com/bitplane/arrayfile)
+
 

{arrayfile-0.0.1 → arrayfile-0.1.0}/README.md

@@ -1,11 +1,11 @@
-# arrayfile
+# 📃 arrayfile
 
 A file-backed numeric array using struct.pack. Does not support inserts or
 slicing.
 
 Smaller than relying on numpy though.
 
-## Installation
+## ▶️ Installation
 
 ```bash
 pip install arrayfile
@@ -13,7 +13,7 @@ pip install arrayfile
 
 ## Usage
 
-### Temporary Array
+### 🗑 Temporary Array
 
 This creates an array in your temp dir:
 
@@ -31,7 +31,7 @@ print(f"Values: {[arr[i] for i in range(len(arr))]}")
 arr.close()  # Clean up resources
 ```
 
-### Persistent Array
+### 💽 Persistent Array
 
 You can use the same file, if you want to persist your data across sessions:
 
@@ -55,7 +55,7 @@ arr.append(2000)
 arr.close()
 ```
 
-## Context manager
+## 🔧 Context manager
 
 It has a finalizer in case you forget to call `close()`, but if you like to keep
 your code tidy, you can use a context manager, like so:
@@ -72,3 +72,18 @@ with Array('d', 'measurements.array', 'w+b') as arr:
         print(f"  {i}: {value:.15f}")
 ```
 
+## ⚖️ License
+
+WTFPL with one extra clause:
+
+1. Don't blame me.
+
+Do what you like, but you're on your own.
+
+## 🔗 Links
+
+* [🏠 home](https://bitplane.net/dev/python/arrayfile)
+* [📖 pydoc](https://bitplane.net/dev/python/arrayfile/pydoc)
+* [🐍 pypi](https://pypi.org/project/arrayfile)
+* [🐱 github](https://github.com/bitplane/arrayfile)
+

{arrayfile-0.0.1 → arrayfile-0.1.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "arrayfile"
 description = "Arrays backed by disk"
-version = "0.0.1"
+version = "0.1.0"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{arrayfile-0.0.1 → arrayfile-0.1.0}/src/arrayfile/array.py

@@ -1,3 +1,4 @@
+import io
 import mmap
 import os
 import struct
@@ -8,6 +9,7 @@ import weakref
 
 class Array:
     CHUNK_SIZE_BYTES = 4096
+    MAX_DTYPE_LENGTH = 8
 
     # Header constants
     MAGIC = b"ARYF"
@@ -20,6 +22,13 @@ class Array:
     def __init__(self, dtype, filename=None, mode="r+b", initial_elements=0):
         self._lock = threading.Lock()
 
+        try:
+            dtype_bytes = dtype.encode("ascii")
+        except (AttributeError, UnicodeEncodeError) as error:
+            raise ValueError("dtype must be an ASCII struct format string") from error
+        if len(dtype_bytes) > self.MAX_DTYPE_LENGTH:
+            raise ValueError(f"dtype must be at most {self.MAX_DTYPE_LENGTH} ASCII bytes")
+
         if filename is None:
             fd, filename = tempfile.mkstemp()
             os.close(fd)
@@ -38,29 +47,46 @@ class Array:
 
         if "w" in mode or not os.path.exists(filename):
             # Create or truncate file
-            self._file = open(filename, "w+b")
+            mode = "w+b"
+            self._file = open(filename, "w+b")  # noqa: SIM115 - kept open for the Array lifetime
             self._len = 0
             self._allocate_capacity(initial_elements)
             self._write_header()
         else:
             # Open existing file
-            self._file = open(filename, mode)
+            self._file = open(filename, mode)  # noqa: SIM115 - kept open for the Array lifetime
             if not self._read_header():
                 raise ValueError("File does not have a valid array header")
 
             current_file_size = os.fstat(self._file.fileno()).st_size
             data_size = current_file_size - self.HEADER_SIZE
+            required_data_size = self._len * self._element_size
+            if required_data_size > data_size:
+                raise ValueError(
+                    f"Array header declares {self._len} elements, but file only contains "
+                    f"{data_size // self._element_size}"
+                )
+
+            self._capacity_bytes = data_size
+            self._capacity = data_size // self._element_size
 
-            # Calculate capacity based on current data size and ensure chunk alignment
-            min_elements = (data_size + self._element_size - 1) // self._element_size
-            self._allocate_capacity(min_elements)
+        self._writable = any(flag in mode for flag in "+wa")
 
         # Only mmap if the file has a non-zero size
         if self._capacity_bytes > 0:
-            self._mmap = mmap.mmap(self._file.fileno(), 0)
+            access = mmap.ACCESS_WRITE if self._writable else mmap.ACCESS_READ
+            self._mmap = mmap.mmap(self._file.fileno(), 0, access=access)
 
         # Set up finalizer to ensure cleanup even if close() isn't called
-        self._finalizer = weakref.finalize(self, self.close)
+        self._finalizer_state = {
+            "dtype": self._dtype,
+            "element_size": self._element_size,
+            "file": self._file,
+            "length": self._len,
+            "mmap": self._mmap,
+            "writable": self._writable,
+        }
+        self._finalizer = weakref.finalize(self, self._finalize_resources, self._finalizer_state)
 
     def __len__(self):
         return self._len
@@ -91,10 +117,42 @@ class Array:
         except struct.error as e:
             raise TypeError(f"Value {value} cannot be packed as {self._dtype_format}: {e}")
 
+    def _ensure_writable(self):
+        if not self._writable:
+            raise io.UnsupportedOperation("Array is not writable")
+
+    @classmethod
+    def _finalize_resources(cls, state):
+        mapped_file = state["mmap"]
+        if mapped_file:
+            mapped_file.flush()
+            mapped_file.close()
+
+        file = state["file"]
+        if not file:
+            return
+
+        if state["writable"]:
+            dtype_bytes = state["dtype"].encode("ascii").ljust(cls.MAX_DTYPE_LENGTH, b"\x00")
+            header = struct.pack(
+                cls.HEADER_FORMAT,
+                cls.MAGIC,
+                cls.HEADER_VERSION,
+                len(state["dtype"]),
+                dtype_bytes,
+                state["element_size"],
+                state["length"],
+            )
+            file.seek(0)
+            file.write(header)
+            file.truncate(cls.HEADER_SIZE + state["length"] * state["element_size"])
+            file.flush()
+
+        file.close()
+
     def _write_header(self):
         """Write header to the beginning of the file."""
-        dtype_bytes = self._dtype.encode("ascii")[:8]  # Limit to 8 bytes
-        dtype_bytes = dtype_bytes.ljust(8, b"\x00")  # Pad with nulls
+        dtype_bytes = self._dtype.encode("ascii").ljust(self.MAX_DTYPE_LENGTH, b"\x00")
 
         header = struct.pack(
             self.HEADER_FORMAT,
@@ -126,6 +184,9 @@ class Array:
         if version != self.HEADER_VERSION:
             raise ValueError(f"Unsupported header version: {version}")
 
+        if dtype_len > self.MAX_DTYPE_LENGTH:
+            raise ValueError(f"File dtype length {dtype_len} exceeds header capacity")
+
         # Extract dtype string
         dtype = dtype_bytes[:dtype_len].decode("ascii")
 
@@ -153,6 +214,7 @@ class Array:
         index = self._validate_index(index)
 
         with self._lock:
+            self._ensure_writable()
             if not self._mmap:
                 raise RuntimeError("Array is not memory-mapped. This should not happen if len > 0.")
 
@@ -162,6 +224,7 @@ class Array:
 
     def append(self, value):
         with self._lock:
+            self._ensure_writable()
             if self._len == self._capacity:
                 self._resize(self._len + 1)
 
@@ -170,6 +233,7 @@ class Array:
 
             self._mmap[offset : offset + self._element_size] = packed_value
             self._len += 1
+            self._finalizer_state["length"] = self._len
 
     def _allocate_capacity(self, min_elements):
         """Allocate capacity for at least min_elements, rounded up to chunk boundary."""
@@ -186,6 +250,7 @@ class Array:
 
         self._allocate_capacity(min_new_len)
         self._mmap = mmap.mmap(self._file.fileno(), 0)
+        self._finalizer_state["mmap"] = self._mmap
 
     def extend(self, iterable):
         values = list(iterable)
@@ -195,6 +260,7 @@ class Array:
             return
 
         with self._lock:
+            self._ensure_writable()
             new_len = self._len + num_new_elements
             if new_len > self._capacity:
                 self._resize(new_len)
@@ -207,6 +273,7 @@ class Array:
                 offset += self._element_size
 
             self._len = new_len
+            self._finalizer_state["length"] = self._len
 
     def __contains__(self, value):
         for i in range(self._len):
@@ -225,15 +292,18 @@ class Array:
             return NotImplemented
 
         with self._lock:
+            self._ensure_writable()
             if value == 0:
                 self._len = 0
                 if self._mmap:
                     self._mmap.close()
                     self._mmap = None
+                    self._finalizer_state["mmap"] = None
                 if self._file:
                     self._file.truncate(0)
                 self._capacity = 0
                 self._capacity_bytes = 0
+                self._finalizer_state["length"] = self._len
             elif value > 1:
                 original_len = self._len
                 new_total_len = original_len * value
@@ -253,31 +323,21 @@ class Array:
                     dst_offset += chunk_size
 
                 self._len = new_total_len
+                self._finalizer_state["length"] = self._len
         return self
 
     def flush(self):
-        if self._mmap:
-            self._mmap.flush()
+        with self._lock:
+            if self._mmap:
+                self._mmap.flush()
+            if self._file and self._writable:
+                self._write_header()
 
     def close(self):
-        if self._mmap:
-            # Ensure all writes are on disk before truncating
-            self._mmap.flush()
-            self._mmap.close()
-            self._mmap = None
-
-        if self._file:
-            # Update header with final length
-            self._write_header()
-
-            # Only truncate if the file was opened in a writable mode
-            # and if the current size is greater than the actual data length
-            current_file_size = os.fstat(self._file.fileno()).st_size
-            actual_total_size = self.HEADER_SIZE + self._len * self._element_size
-            if current_file_size > actual_total_size:
-                self._file.truncate(actual_total_size)
-            self._file.close()
-            self._file = None
+        if self._finalizer.alive:
+            self._finalizer()
+        self._mmap = None
+        self._file = None
 
     def __enter__(self):
         return self