arrayfile 0.0.1__tar.gz

arrayfile-0.0.1/LICENSE.md

@@ -0,0 +1,7 @@
+# WTFPL + Warranty
+
+Licensed under the WTFPL with one additional clause:
+
+1. Don't blame me.
+
+Do whatever the fuck you want, just don't blame me.

arrayfile-0.0.1/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: arrayfile
+Version: 0.0.1
+Summary: Arrays backed by disk
+Keywords: logging,terminal,scrollback,indexing
+Author-email: Gareth Davidson <gaz@bitplane.net>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: System :: Logging
+Classifier: Topic :: Terminals
+License-File: LICENSE.md
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pytest ; extra == "dev"
+Requires-Dist: coverage ; extra == "dev"
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: build ; extra == "dev"
+Requires-Dist: twine ; extra == "dev"
+Requires-Dist: ruff ; extra == "dev"
+Requires-Dist: pydoc-markdown ; extra == "dev"
+Provides-Extra: dev
+
+# arrayfile
+
+A file-backed numeric array using struct.pack. Does not support inserts or
+slicing.
+
+Smaller than relying on numpy though.
+
+## Installation
+
+```bash
+pip install arrayfile
+```
+
+## Usage
+
+### Temporary Array
+
+This creates an array in your temp dir:
+
+```python
+from arrayfile import Array
+
+# Create a temporary array with float data
+arr = Array('f')
+arr.append(3.14)
+arr.append(2.71)
+arr.extend([1.41, 1.73])
+
+print(f"Length: {len(arr)}")
+print(f"Values: {[arr[i] for i in range(len(arr))]}")
+arr.close()  # Clean up resources
+```
+
+### Persistent Array
+
+You can use the same file, if you want to persist your data across sessions:
+
+```python
+from arrayfile import Array
+
+# Create and populate an array file
+arr = Array('i', 'numbers.array', 'w+b')
+for i in range(1000):
+    arr.append(i * 2)
+arr.close()
+
+# Reopen the same file later
+arr = Array('i', 'numbers.array', 'r+b')
+print(f"Array has {len(arr)} elements")
+print(f"First element: {arr[0]}")
+print(f"Last element: {arr[-1]}")
+
+# Add more data
+arr.append(2000)
+arr.close()
+```
+
+## Context manager
+
+It has a finalizer in case you forget to call `close()`, but if you like to keep
+your code tidy, you can use a context manager, like so:
+
+```python
+from arrayfile import Array
+
+# Using double precision floats with context manager
+with Array('d', 'measurements.array', 'w+b') as arr:
+    arr.extend([3.141592653589793, 2.718281828459045, 1.4142135623730951])
+
+    print(f"Stored {len(arr)} precise measurements")
+    for i, value in enumerate(arr):
+        print(f"  {i}: {value:.15f}")
+```
+
+

arrayfile-0.0.1/README.md

@@ -0,0 +1,74 @@
+# arrayfile
+
+A file-backed numeric array using struct.pack. Does not support inserts or
+slicing.
+
+Smaller than relying on numpy though.
+
+## Installation
+
+```bash
+pip install arrayfile
+```
+
+## Usage
+
+### Temporary Array
+
+This creates an array in your temp dir:
+
+```python
+from arrayfile import Array
+
+# Create a temporary array with float data
+arr = Array('f')
+arr.append(3.14)
+arr.append(2.71)
+arr.extend([1.41, 1.73])
+
+print(f"Length: {len(arr)}")
+print(f"Values: {[arr[i] for i in range(len(arr))]}")
+arr.close()  # Clean up resources
+```
+
+### Persistent Array
+
+You can use the same file, if you want to persist your data across sessions:
+
+```python
+from arrayfile import Array
+
+# Create and populate an array file
+arr = Array('i', 'numbers.array', 'w+b')
+for i in range(1000):
+    arr.append(i * 2)
+arr.close()
+
+# Reopen the same file later
+arr = Array('i', 'numbers.array', 'r+b')
+print(f"Array has {len(arr)} elements")
+print(f"First element: {arr[0]}")
+print(f"Last element: {arr[-1]}")
+
+# Add more data
+arr.append(2000)
+arr.close()
+```
+
+## Context manager
+
+It has a finalizer in case you forget to call `close()`, but if you like to keep
+your code tidy, you can use a context manager, like so:
+
+```python
+from arrayfile import Array
+
+# Using double precision floats with context manager
+with Array('d', 'measurements.array', 'w+b') as arr:
+    arr.extend([3.141592653589793, 2.718281828459045, 1.4142135623730951])
+
+    print(f"Stored {len(arr)} precise measurements")
+    for i, value in enumerate(arr):
+        print(f"  {i}: {value:.15f}")
+```
+

arrayfile-0.0.1/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "arrayfile"
+description = "Arrays backed by disk"
+version = "0.0.1"
+authors = [
+    { name = "Gareth Davidson", email = "gaz@bitplane.net" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "CC0"} # wtfpl
+keywords = ["logging", "terminal", "scrollback", "indexing"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Topic :: System :: Logging",
+    "Topic :: Terminals",
+]
+
+dependencies = [
+]
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit",
+    "pytest",
+    "coverage",
+    "pytest-cov",
+    "build",
+    "twine",
+    "ruff",
+    "pydoc-markdown"
+]
+
+[build-system]
+build-backend = "flit_core.buildapi"
+requires = ["flit_core >=3.2,<4"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.format]
+docstring-code-format = true

arrayfile-0.0.1/src/arrayfile/__init__.py

@@ -0,0 +1,5 @@
+"""arrayfile - Memory-mapped array implementation for efficient file-backed arrays."""
+
+from .array import Array
+
+__all__ = ["Array"]

arrayfile-0.0.1/src/arrayfile/array.py

@@ -0,0 +1,286 @@
+import mmap
+import os
+import struct
+import tempfile
+import threading
+import weakref
+
+
+class Array:
+    CHUNK_SIZE_BYTES = 4096
+
+    # Header constants
+    MAGIC = b"ARYF"
+    HEADER_VERSION = 1
+    HEADER_SIZE = 32
+    HEADER_FORMAT = (
+        "<4sHB8sIQ5x"  # magic(4), version(2), dtype_len(1), dtype(8), element_size(4), length(8), reserved(5)
+    )
+
+    def __init__(self, dtype, filename=None, mode="r+b", initial_elements=0):
+        self._lock = threading.Lock()
+
+        if filename is None:
+            fd, filename = tempfile.mkstemp()
+            os.close(fd)
+            mode = "w+b"  # Always create new temp files
+
+        self._filename = filename
+        self._dtype = dtype
+        self._dtype_format = dtype
+        self._element_size = struct.calcsize(dtype)
+        self._file = None
+        self._mmap = None
+        self._len = 0
+        self._capacity = 0
+        self._capacity_bytes = 0  # Initialize _capacity_bytes here
+        self._data_offset = self.HEADER_SIZE  # All data starts after header
+
+        if "w" in mode or not os.path.exists(filename):
+            # Create or truncate file
+            self._file = open(filename, "w+b")
+            self._len = 0
+            self._allocate_capacity(initial_elements)
+            self._write_header()
+        else:
+            # Open existing file
+            self._file = open(filename, mode)
+            if not self._read_header():
+                raise ValueError("File does not have a valid array header")
+
+            current_file_size = os.fstat(self._file.fileno()).st_size
+            data_size = current_file_size - self.HEADER_SIZE
+
+            # Calculate capacity based on current data size and ensure chunk alignment
+            min_elements = (data_size + self._element_size - 1) // self._element_size
+            self._allocate_capacity(min_elements)
+
+        # Only mmap if the file has a non-zero size
+        if self._capacity_bytes > 0:
+            self._mmap = mmap.mmap(self._file.fileno(), 0)
+
+        # Set up finalizer to ensure cleanup even if close() isn't called
+        self._finalizer = weakref.finalize(self, self.close)
+
+    def __len__(self):
+        return self._len
+
+    def __iter__(self):
+        current_len = self._len
+        for i in range(current_len):
+            yield self[i]
+
+    def _validate_index(self, index):
+        """Validate and normalize an index, returning the normalized value."""
+        if not isinstance(index, int):
+            raise TypeError("Index must be an integer")
+
+        # Handle negative indices
+        if index < 0:
+            index = self._len + index
+
+        if not (0 <= index < self._len):
+            raise IndexError("Index out of bounds")
+
+        return index
+
+    def _pack_value(self, value):
+        """Pack a value into bytes according to the dtype format."""
+        try:
+            return struct.pack(self._dtype_format, value)
+        except struct.error as e:
+            raise TypeError(f"Value {value} cannot be packed as {self._dtype_format}: {e}")
+
+    def _write_header(self):
+        """Write header to the beginning of the file."""
+        dtype_bytes = self._dtype.encode("ascii")[:8]  # Limit to 8 bytes
+        dtype_bytes = dtype_bytes.ljust(8, b"\x00")  # Pad with nulls
+
+        header = struct.pack(
+            self.HEADER_FORMAT,
+            self.MAGIC,
+            self.HEADER_VERSION,
+            len(self._dtype),
+            dtype_bytes,
+            self._element_size,
+            self._len,
+        )
+
+        self._file.seek(0)
+        self._file.write(header)
+        self._file.flush()
+
+    def _read_header(self):
+        """Read and validate header from file. Returns True if valid header, False if no header."""
+        self._file.seek(0)
+        header_data = self._file.read(self.HEADER_SIZE)
+
+        if len(header_data) < self.HEADER_SIZE:
+            return False
+
+        magic, version, dtype_len, dtype_bytes, element_size, length = struct.unpack(self.HEADER_FORMAT, header_data)
+
+        if magic != self.MAGIC:
+            return False
+
+        if version != self.HEADER_VERSION:
+            raise ValueError(f"Unsupported header version: {version}")
+
+        # Extract dtype string
+        dtype = dtype_bytes[:dtype_len].decode("ascii")
+
+        # Validate dtype matches
+        if dtype != self._dtype:
+            raise ValueError(f"File dtype '{dtype}' does not match requested dtype '{self._dtype}'")
+
+        if element_size != self._element_size:
+            raise ValueError(f"File element size {element_size} does not match expected {self._element_size}")
+
+        self._len = length
+        return True
+
+    def __getitem__(self, index):
+        index = self._validate_index(index)
+
+        if not self._mmap:
+            raise RuntimeError("Array is not memory-mapped. This should not happen if len > 0.")
+
+        offset = self._data_offset + index * self._element_size
+        data = self._mmap[offset : offset + self._element_size]
+        return struct.unpack(self._dtype_format, data)[0]
+
+    def __setitem__(self, index, value):
+        index = self._validate_index(index)
+
+        with self._lock:
+            if not self._mmap:
+                raise RuntimeError("Array is not memory-mapped. This should not happen if len > 0.")
+
+            offset = self._data_offset + index * self._element_size
+            packed_value = self._pack_value(value)
+            self._mmap[offset : offset + self._element_size] = packed_value
+
+    def append(self, value):
+        with self._lock:
+            if self._len == self._capacity:
+                self._resize(self._len + 1)
+
+            offset = self._data_offset + self._len * self._element_size
+            packed_value = self._pack_value(value)
+
+            self._mmap[offset : offset + self._element_size] = packed_value
+            self._len += 1
+
+    def _allocate_capacity(self, min_elements):
+        """Allocate capacity for at least min_elements, rounded up to chunk boundary."""
+        bytes_needed = min_elements * self._element_size + self.HEADER_SIZE
+        chunks_needed = (bytes_needed + self.CHUNK_SIZE_BYTES - 1) // self.CHUNK_SIZE_BYTES
+        total_file_size = chunks_needed * self.CHUNK_SIZE_BYTES
+        self._capacity_bytes = total_file_size - self.HEADER_SIZE
+        self._capacity = self._capacity_bytes // self._element_size
+        self._file.truncate(total_file_size)
+
+    def _resize(self, min_new_len):
+        if self._mmap:
+            self._mmap.close()
+
+        self._allocate_capacity(min_new_len)
+        self._mmap = mmap.mmap(self._file.fileno(), 0)
+
+    def extend(self, iterable):
+        values = list(iterable)
+        num_new_elements = len(values)
+
+        if num_new_elements == 0:
+            return
+
+        with self._lock:
+            new_len = self._len + num_new_elements
+            if new_len > self._capacity:
+                self._resize(new_len)
+
+            # Batch write all values directly to mmap
+            offset = self._data_offset + self._len * self._element_size
+            for value in values:
+                packed_value = self._pack_value(value)
+                self._mmap[offset : offset + self._element_size] = packed_value
+                offset += self._element_size
+
+            self._len = new_len
+
+    def __contains__(self, value):
+        for i in range(self._len):
+            if self[i] == value:
+                return True
+        return False
+
+    def __iadd__(self, other):
+        if hasattr(other, "__iter__"):
+            self.extend(other)
+            return self
+        return NotImplemented
+
+    def __imul__(self, value):
+        if not isinstance(value, int) or value < 0:
+            return NotImplemented
+
+        with self._lock:
+            if value == 0:
+                self._len = 0
+                if self._mmap:
+                    self._mmap.close()
+                    self._mmap = None
+                if self._file:
+                    self._file.truncate(0)
+                self._capacity = 0
+                self._capacity_bytes = 0
+            elif value > 1:
+                original_len = self._len
+                new_total_len = original_len * value
+
+                # Resize if needed
+                if new_total_len > self._capacity:
+                    self._resize(new_total_len)
+
+                # Copy data in-place
+                src_offset = self._data_offset
+                dst_offset = self._data_offset + original_len * self._element_size
+
+                for _ in range(value - 1):
+                    # Copy the original data chunk
+                    chunk_size = original_len * self._element_size
+                    self._mmap[dst_offset : dst_offset + chunk_size] = self._mmap[src_offset : src_offset + chunk_size]
+                    dst_offset += chunk_size
+
+                self._len = new_total_len
+        return self
+
+    def flush(self):
+        if self._mmap:
+            self._mmap.flush()
+
+    def close(self):
+        if self._mmap:
+            # Ensure all writes are on disk before truncating
+            self._mmap.flush()
+            self._mmap.close()
+            self._mmap = None
+
+        if self._file:
+            # Update header with final length
+            self._write_header()
+
+            # Only truncate if the file was opened in a writable mode
+            # and if the current size is greater than the actual data length
+            current_file_size = os.fstat(self._file.fileno()).st_size
+            actual_total_size = self.HEADER_SIZE + self._len * self._element_size
+            if current_file_size > actual_total_size:
+                self._file.truncate(actual_total_size)
+            self._file.close()
+            self._file = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()