solentware-base 5.0__tar.gz → 5.2__tar.gz

solentware-base-5.2/MANIFEST.in

@@ -0,0 +1,2 @@
+# MANIFEST.in exists to get tox.ini included in sdist distributions.
+include tox.ini

solentware-base-5.2/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.1
+Name: solentware-base
+Version: 5.2
+Summary: Bitmapped record number databases.
+Author-email: Roger Marsh <roger.marsh@solentware.co.uk>
+License: Licence for solentware_base package
+        Copyright (c) 2009, Roger Marsh
+        All rights reserved.
+        
+        Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+        
+        
+        Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+        
+        Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+        
+        Neither the name of the owner nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+        
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, http://www.solentware.co.uk
+Project-URL: Repository, https://github.com/RogerMarsh/solentware-base.git
+Project-URL: Bug Tracker, https://github.com/RogerMarsh/solentware-base/issues
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Database
+Classifier: Intended Audience :: Developers
+Classifier: Development Status :: 3 - Alpha
+License-File: LICENCE
+
+=========================================================================
+A bitmapped record number database interface for various database engines
+=========================================================================
+
+.. contents::
+
+
+Description
+===========
+
+A record number database is seen as a set of fixed size segments, and an existence bitmap is associated with each segment which contains any records.  The idea is from Model 204 (Rocket Software).
+
+Where an index value refers to many records in a segment these references are encoded as a bitmap or a list of record numbers, in a single <index value> : <bitmap or list> record, rather than as many <index value> : <record number> records.  When 'many' is small a list is favoured to save the space needed for large numbers of unset bits.
+
+The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, lmdb, and sqlite3, Python modules are supported.
+
+The interfaces provided by the dbm.gnu, dbm.ndbm, unqlite, and vedis, Python modules are available for experiment.
+
+
+Installation Instructions
+=========================
+
+This package is available on PyPI so can be installed by typing
+
+   python -m pip install --user solentware-base
+
+at the command prompt.
+
+See `Python`_ and `pip`_ documentation for alternatives and platform-specific points.
+
+
+.. _Python: https://python.org
+.. _pip: https://packaging.python.org/installing

{solentware-base-5.0 → solentware-base-5.2}/README

@@ -12,7 +12,7 @@ A record number database is seen as a set of fixed size segments, and an existen
 
 Where an index value refers to many records in a segment these references are encoded as a bitmap or a list of record numbers, in a single <index value> : <bitmap or list> record, rather than as many <index value> : <record number> records.  When 'many' is small a list is favoured to save the space needed for large numbers of unset bits.
 
-The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, and sqlite3, Python modules are supported.
+The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, lmdb, and sqlite3, Python modules are supported.
 
 The interfaces provided by the dbm.gnu, dbm.ndbm, unqlite, and vedis, Python modules are available for experiment.
 

solentware-base-5.2/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 79
+
+[project]
+name = "solentware-base"
+version = "5.2"
+authors = [
+  { name="Roger Marsh", email="roger.marsh@solentware.co.uk" },
+]
+description = "Bitmapped record number databases."
+classifiers = [
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Operating System :: OS Independent",
+    "Topic :: Database",
+    "Intended Audience :: Developers",
+    "Development Status :: 3 - Alpha",
+]
+license = {file = "LICENCE"}
+readme = "README"
+
+[project.urls]
+Homepage = "http://www.solentware.co.uk"
+Repository = "https://github.com/RogerMarsh/solentware-base.git"
+"Bug Tracker" = "https://github.com/RogerMarsh/solentware-base/issues"
+
+[tool.setuptools]
+packages = [
+    "solentware_base",
+    "solentware_base.core",
+    "solentware_base.tools",
+]

solentware-base-5.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

{solentware-base-5.0 → solentware-base-5.2}/solentware_base/core/_bytebit.py

@@ -72,6 +72,7 @@ class Bitarray:
 
         """
         # Time taken proportional to number of non-zero bytes.
+        del value
         return sum(self.bitarray_bytes.translate(_bits_count, b"\x00"))
 
     def frombytes(self, from_):
@@ -118,8 +119,8 @@ class Bitarray:
                     ):
                         if self.bitarray_bytes[start_byte] & 128 >> bit:
                             return 8 * start_byte + bit
-            except IndexError:
-                raise ValueError("Set bit (True) not found")
+            except IndexError as exc:
+                raise ValueError("Set bit (True) not found") from exc
             for byte in range(1 + start_byte, stop_byte):
                 if self.bitarray_bytes[byte] != 0:
                     return 8 * byte + _first_set_bit[self.bitarray_bytes[byte]]
@@ -137,8 +138,8 @@ class Bitarray:
                 ):
                     if not self.bitarray_bytes[start_byte] & 128 >> bit:
                         return 8 * start_byte + bit
-        except IndexError:
-            raise ValueError("Unset bit (False) not found")
+        except IndexError as exc:
+            raise ValueError("Unset bit (False) not found") from exc
         for byte in range(1 + start_byte, stop_byte):
             if self.bitarray_bytes[byte] != 255:
                 for bit in range(0, 8):
@@ -183,6 +184,7 @@ class Bitarray:
         in bitarray-0.8.1 is search(bitarray('1')).
 
         """
+        del bitarray, limit
         bitscan = []
         for j, byte in enumerate(self.bitarray_bytes):
             if not byte:

{solentware-base-5.0 → solentware-base-5.2}/solentware_base/core/_database.py

@@ -2,7 +2,7 @@
 # Copyright 2008, 2019 Roger Marsh
 # Licence: See LICENCE (BSD licence)
 
-"""Define the database interface shared by _db and _sqlite modules.
+"""Define the interface shared by python database modules.
 
 The major components are the delete_instance, edit_instance, and put_instance,
 methods.  The _dpt module's versions of these methods are too different to
@@ -24,8 +24,53 @@ from .recordset import (
 )
 
 
+class DataSourceCursorError(Exception):
+    """Exception for database-specific DataSourceCursor setting."""
+
+
+# The Database class was added to the ignored-classes list of pylint.conf for
+# solentware_base at a time when all the no-member errors (E1101) referred
+# to it or one of it's subclasses with the same name in another module.
 class Database:
-    """Define file and record access methods; which subclasses may override."""
+    """Define file and record access methods; which subclasses may override.
+
+    The set_segment_size() method is used to set the segment size to fit
+    the segment size recorded on a database.  There is no segment_size_bytes
+    class attribute.
+
+    Property file_per_database returns False because all 'key:value' sets,
+    and the associated inverted list indicies, are held in one file by
+    default.  Berkeley DB is the known example where it is reasonable to
+    return True: override the class attribute, if this case is wanted.
+
+    """
+
+    _file_per_database = False
+
+    @property
+    def file_per_database(self):
+        """Return True if each database is in a separate file.
+
+        Berkeley DB is the known cases where True is reasonable (see ._db
+        module), and DPT is the known case where True is mandatory (see
+        ._dpt module).
+
+        """
+        return self._file_per_database
+
+    def _generate_database_file_name(self, name):
+        """Return path to database file.
+
+        By default this is a file where os.path.basename(path) is same as
+        os.path.dirname(path); and path is assumed to follow this rule.
+
+        However some database engines mangle os.path.basename(path) to
+        give the file name.  Override this method if necessary to apply
+        the required mangle to os.path.basename(path).
+
+        """
+        del name
+        return self.database_file
 
     def delete_instance(self, dbset, instance):
         """Delete an existing instance on databases in dbset.
@@ -90,7 +135,6 @@ class Database:
             self.delete(dbset, oldkey, instance.srvalue)
             key = self.put(dbset, newkey, instance.newrecord.srvalue)
             if key is not None:
-
                 # put was append to record number database and
                 # returned the new primary key. Adjust record key
                 # for secondary updates.
@@ -180,7 +224,6 @@ class Database:
         putkey = instance.key.pack()
         instance.set_packed_value_and_indexes()
         if putkey is None:
-
             # reuse record number if possible
             putkey = self.get_lowest_freed_record_number(dbset)
             if putkey is not None:
@@ -188,7 +231,6 @@ class Database:
 
         key = self.put(dbset, putkey, instance.srvalue)
         if key is not None:
-
             # put was append to record number database and
             # returned the new primary key. Adjust record key
             # for secondary updates.
@@ -229,6 +271,7 @@ class Database:
 
     def make_segment(self, key, segment_number, record_count, records):
         """Return a Segment subclass instance created from arguments."""
+        del key  # Looks reasonable to remove key from arguments.
         if record_count == 1:
             return RecordsetSegmentInt(
                 segment_number,
@@ -300,6 +343,7 @@ class Database:
         *a absorbs the arguments needed by the DPT version of this method.
 
         """
+        del a
         return oldcursor
 
     def allocate_and_open_contexts(self, files=None):
@@ -311,6 +355,7 @@ class Database:
         The Berkeley DB DBEnv or SQLite3 Connection object is assumed closed as
         well so open all files in the specification.
         """
+        del files
         self.open_database()
 
     def open_database_contexts(self, files=None):
@@ -329,6 +374,76 @@ class Database:
     def end_read_only_transaction(self):
         """Do nothing, present for compatibility with Symas LMMD."""
 
+    def increase_database_record_capacity(self, **kwargs):
+        """Do nothing, present for compatibility with DPT."""
+        del kwargs
+
+    def get_database_table_sizes(self, **kwargs):
+        """Return empty dict, present for compatibility with DPT."""
+        del kwargs
+        return {}
+
+    def close_datasourcecursor_recordset(self, datasourcecursor):
+        """Do nothing.  Override if necessary.
+
+        Present for compatibility with DPT database engine.
+        """
+        del datasourcecursor
+
+    def set_datasourcecursor_recordset(self, datasourcecursor, recordset):
+        """Validate and set recordset as datasourcecursor's recordset.
+
+        The recordset and datasourcecursor must be associated with the same
+        database identity and be the identity of self.
+
+        This method exists because in DPT these checks are implied when the
+        recordset and cursor are created, and both objects are details of
+        the implementation.
+        """
+        if datasourcecursor.dbhome is not self:
+            raise DataSourceCursorError("DataSource is not for this database")
+        if datasourcecursor.recordset:
+            if (
+                datasourcecursor.recordset.dbidentity
+                == recordset.recordset.dbidentity
+            ):
+                datasourcecursor.recordset.recordset.close()
+                datasourcecursor.recordset = recordset
+            else:
+                raise DataSourceCursorError(
+                    "New and existing Recordsets are for different databases"
+                )
+        elif datasourcecursor.dbidentity == recordset.recordset.dbidentity:
+            datasourcecursor.recordset = recordset
+        else:
+            raise DataSourceCursorError(
+                "New Recordset and DataSource are for different databases"
+            )
+
+    def get_datasourcecursor_recordset_cursor(self, dsc):
+        """Create and return cursor on dsc's recordset.
+
+        dsc not datasourcecursor to shorten argument name.
+
+        """
+        if dsc.dbhome is not self:
+            raise DataSourceCursorError("DataSource is not for this database")
+        if dsc.recordset:
+            if dsc.dbidentity == dsc.recordset.recordset.dbidentity:
+                cursor = dsc.recordset.dbhome.create_recordset_cursor(
+                    dsc.recordset.recordset
+                )
+            else:
+                raise DataSourceCursorError(
+                    "Recordset and DataSource are for different databases"
+                )
+        else:
+            dsc.recordset = dsc.dbhome.recordlist_nil(dsc.dbset)
+            cursor = dsc.recordset.dbhome.create_recordset_cursor(
+                dsc.recordset.recordset
+            )
+        return cursor
+
 
 class ExistenceBitmapControl:
     """Base class for managing existence bitmap of file in database.

solentware-base-5.2/solentware_base/core/_databasedu.py

@@ -0,0 +1,272 @@
+# _databasedu.py
+# Copyright 2008, 2019 Roger Marsh
+# Licence: See LICENCE (BSD licence)
+
+"""Define the database interface for deferred updates.
+
+The major component is the put_instance method.  The DPT database way of doing
+deferred updates is so different there is no equivalent separate module for
+DPT.
+
+"""
+from . import _database
+from .segmentsize import SegmentSize
+from .constants import SECONDARY
+from .bytebit import Bitarray
+from . import merge
+
+
+class DatabaseduError(Exception):
+    """Exception for Database class."""
+
+
+class Database(_database.Database):
+    """Provide deferred update versions of the record update methods."""
+
+    def put_instance(self, dbset, instance):
+        """Put new instance on database dbset.
+
+        This method assumes all primary databases are integer primary key,
+        and there is enough memory to do a segment at a time.
+
+        """
+        putkey = instance.key.pack()
+        instance.set_packed_value_and_indexes()
+        if putkey is not None:
+            # reuse record number is not allowed
+            raise DatabaseduError(
+                "Cannot reuse record number in deferred update."
+            )
+        key = self.put(dbset, putkey, instance.srvalue)
+
+        # put was append to record number database and
+        # returned the new primary key. Adjust record key
+        # for secondary updates.
+        instance.key.load(key)
+        putkey = key
+
+        instance.srkey = self.encode_record_number(putkey)
+        srindex = instance.srindex
+        segment, record_number = divmod(putkey, SegmentSize.db_segment_size)
+        self._defer_add_record_to_ebm(dbset, segment, record_number)
+        pcb = instance.putcallbacks
+        for secondary in srindex:
+            if secondary not in self.specification[dbset][SECONDARY]:
+                if secondary in pcb:
+                    pcb[secondary](instance, srindex[secondary])
+                continue
+            for j in srindex[secondary]:
+                self._defer_add_record_to_field_value(
+                    dbset, secondary, j, segment, record_number
+                )
+
+        if record_number in self.deferred_update_points:
+            self._write_existence_bit_map(dbset, segment)
+            for secondary in self.specification[dbset][SECONDARY]:
+                self.sort_and_write(dbset, secondary, segment)
+            if record_number == max(self.deferred_update_points):
+                self.first_chunk[dbset] = True
+            elif record_number == min(self.deferred_update_points):
+                self.first_chunk[dbset] = False
+                self.high_segment[dbset] = segment
+
+    def index_instance(self, dbset, instance):
+        """Apply instance index values on database dbset.
+
+        This method assumes all primary databases are integer primary key,
+        and there is enough memory to do a segment at a time.
+
+        """
+        putkey = instance.key.pack()
+        if putkey is None:
+            # Apply index values without record number is not allowed.
+            raise DatabaseduError(
+                "Cannot apply index values without a record number"
+            )
+        instance.set_packed_value_and_indexes()
+        instance.srkey = self.encode_record_number(putkey)
+        srindex = instance.srindex
+        segment, record_number = divmod(putkey, SegmentSize.db_segment_size)
+        pcb = instance.putcallbacks
+        for secondary in srindex:
+            if secondary not in self.specification[dbset][SECONDARY]:
+                if secondary in pcb:
+                    pcb[secondary](instance, srindex[secondary])
+                continue
+            for j in srindex[secondary]:
+                self._defer_add_record_to_field_value(
+                    dbset, secondary, j, segment, record_number
+                )
+
+        if record_number in self.deferred_update_points:
+            for secondary in srindex:
+                assert secondary in self.specification[dbset][SECONDARY]
+                self.sort_and_write(dbset, secondary, segment)
+            if record_number == max(self.deferred_update_points):
+                self.first_chunk[dbset] = True
+            elif record_number == min(self.deferred_update_points):
+                self.first_chunk[dbset] = False
+                self.high_segment[dbset] = segment
+
+    def _defer_add_record_to_ebm(self, file, segment, record_number):
+        """Add bit to existence bit map for new record and defer update."""
+        assert file in self.specification
+        try:
+            # Assume cached segment existence bit map exists
+            self.existence_bit_maps[file][segment][record_number] = True
+        except KeyError:
+            # Get the segment existence bit map from database
+            ebmb = self.get_ebm_segment(self.ebm_control[file], segment)
+            if ebmb is None:
+                # It does not exist so create a new empty one
+                ebm = SegmentSize.empty_bitarray.copy()
+            else:
+                # It does exist so convert database representation to bitarray
+                ebm = Bitarray()
+                ebm.frombytes(ebmb)
+            # Set bit for record number and add segment to cache
+            ebm[record_number] = True
+            if file not in self.existence_bit_maps:
+                self.existence_bit_maps[file] = {}
+            self.existence_bit_maps[file][segment] = ebm
+
+    def _defer_add_record_to_field_value(
+        self, file, field, key, segment, record_number
+    ):
+        """Add record_number to cached segment for key."""
+        del segment
+        assert file in self.specification
+        try:
+            value_segments = self.value_segments[file][field]
+        except KeyError:
+            value_segments = self.value_segments.setdefault(
+                file, {}
+            ).setdefault(field, {})
+        values = value_segments.get(key)
+        if values is None:
+            value_segments[key] = [record_number]
+        elif isinstance(values, list):
+            # A (value, record_number) can be given many times.
+            # Ensure a record_number appears in the list once only.
+            if values[-1] != record_number:
+                values.append(record_number)
+                if len(values) > SegmentSize.db_upper_conversion_limit:
+                    vsk = value_segments[
+                        key
+                    ] = SegmentSize.empty_bitarray.copy()
+                    for j in values:
+                        vsk[j] = True
+                    vsk[record_number] = True
+
+        else:
+            values[record_number] = True
+
+    def _prepare_segment_record_list(self, file, field):
+        """Convert dict of record number lists to database record format.
+
+        A single record in a segment for an index value is represented
+        as a number within the segment.
+
+        Multiple records are represented as lists of record numbers or
+        bitmaps depending on how many are in the segment.
+
+        """
+        # Lookup table is much quicker, and noticeable, in bulk use.
+        int_to_bytes = self._int_to_bytes
+
+        segvalues = self.value_segments[file][field]
+        for k in segvalues:
+            value = segvalues[k]
+            if isinstance(value, list):
+                # A single record is presented as an integer: the
+                # database engine will decide the transformation.
+                if len(value) == 1:
+                    segvalues[k] = [1, value[-1]]
+                else:
+                    segvalues[k] = [
+                        len(value),
+                        b"".join([int_to_bytes[n] for n in value]),
+                    ]
+
+            else:
+                segvalues[k] = [
+                    value.count(),
+                    value.tobytes(),
+                ]
+
+    def set_segment_size(self):
+        """Extend and set a deferred update point at end of segment."""
+        super().set_segment_size()
+
+        # Override in subclasses if more frequent deferred update is required.
+        self.deferred_update_points = frozenset(
+            [SegmentSize.db_segment_size - 1]
+        )
+
+    def deferred_update_housekeeping(self):
+        """Do nothing.  Subclasses should override this method as required.
+
+        Actions are specific to a database engine.
+
+        """
+
+    def encode_number_for_sequential_file_dump(self, number, bytes_):
+        """Return big-endian encoding of number in bytes_ bytes."""
+        return number.to_bytes(bytes_, byteorder="big")
+
+    def encode_segment_for_sequential_file_dump(self, record_numbers):
+        """Return encoding of record numbers appropriate to record count."""
+        if len(record_numbers) > SegmentSize.db_upper_conversion_limit:
+            seg = SegmentSize.empty_bitarray.copy()
+            for bit in record_numbers:
+                seg[bit] = True
+            return seg.tobytes()
+        int_to_bytes = self._int_to_bytes
+        return b"".join([int_to_bytes[n] for n in record_numbers])
+
+    def set_int_to_bytes_lookup(self, lookup=True):
+        """Create lookup to convert integers to bytestring big-endian.
+
+        Set lookup table to None if bool(lookup) is False.
+
+        """
+        if lookup:
+            self._int_to_bytes = [
+                n.to_bytes(2, byteorder="big")
+                for n in range(SegmentSize.db_segment_size)
+            ]
+        else:
+            self._int_to_bytes = None
+
+    def merge_import(self, index_directory, file, field, commit_limit):
+        """Yield count of sorted items written to an index at intervals.
+
+        Sorted items from files in index_directory are written to index
+        field in file, and the count of items added is yielded every
+        commit_limit items.
+
+        """
+        writer = self.merge_writer(file, field)
+        merger = merge.Merge(index_directory)
+        commit_count = None
+        for commit_count, item in enumerate(merger.sorter()):
+            if not commit_count % commit_limit:
+                if commit_count:
+                    writer.close_cursor()
+                    self.commit()
+                    self.deferred_update_housekeeping()
+                    yield commit_count
+                    self.start_transaction()
+                    writer.make_new_cursor()
+            writer.write(item)
+        writer.close_cursor()
+        if commit_count is not None:
+            self.commit()
+            self.deferred_update_housekeeping()
+            self.start_transaction()
+
+    def next_sorted_item(self, index_directory):
+        """Yield sorted items from files in index_directory."""
+        merger = merge.Merge(index_directory)
+        for item in merger.sorter():
+            yield item