solentware-base 5.1__tar.gz → 5.2__tar.gz

{solentware-base-5.1/solentware_base.egg-info → solentware-base-5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: solentware-base
-Version: 5.1
+Version: 5.2
 Summary: Bitmapped record number databases.
 Author-email: Roger Marsh <roger.marsh@solentware.co.uk>
 License: Licence for solentware_base package
@@ -23,7 +23,6 @@ Project-URL: Homepage, http://www.solentware.co.uk
 Project-URL: Repository, https://github.com/RogerMarsh/solentware-base.git
 Project-URL: Bug Tracker, https://github.com/RogerMarsh/solentware-base/issues
 Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -48,7 +47,7 @@ A record number database is seen as a set of fixed size segments, and an existen
 
 Where an index value refers to many records in a segment these references are encoded as a bitmap or a list of record numbers, in a single <index value> : <bitmap or list> record, rather than as many <index value> : <record number> records.  When 'many' is small a list is favoured to save the space needed for large numbers of unset bits.
 
-The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, and sqlite3, Python modules are supported.
+The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, lmdb, and sqlite3, Python modules are supported.
 
 The interfaces provided by the dbm.gnu, dbm.ndbm, unqlite, and vedis, Python modules are available for experiment.
 

{solentware-base-5.1 → solentware-base-5.2}/README

@@ -12,7 +12,7 @@ A record number database is seen as a set of fixed size segments, and an existen
 
 Where an index value refers to many records in a segment these references are encoded as a bitmap or a list of record numbers, in a single <index value> : <bitmap or list> record, rather than as many <index value> : <record number> records.  When 'many' is small a list is favoured to save the space needed for large numbers of unset bits.
 
-The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, and sqlite3, Python modules are supported.
+The interfaces provided by the apsw, berkeleydb, bsddb3, dptdb, lmdb, and sqlite3, Python modules are supported.
 
 The interfaces provided by the dbm.gnu, dbm.ndbm, unqlite, and vedis, Python modules are available for experiment.
 

{solentware-base-5.1 → solentware-base-5.2}/pyproject.toml

@@ -10,14 +10,13 @@ line-length = 79
 
 [project]
 name = "solentware-base"
-version = "5.1"
+version = "5.2"
 authors = [
   { name="Roger Marsh", email="roger.marsh@solentware.co.uk" },
 ]
 description = "Bitmapped record number databases."
 classifiers = [
     "License :: OSI Approved :: BSD License",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",

{solentware-base-5.1 → solentware-base-5.2}/solentware_base/core/_database.py

@@ -24,6 +24,10 @@ from .recordset import (
 )
 
 
+class DataSourceCursorError(Exception):
+    """Exception for database-specific DataSourceCursor setting."""
+
+
 # The Database class was added to the ignored-classes list of pylint.conf for
 # solentware_base at a time when all the no-member errors (E1101) referred
 # to it or one of it's subclasses with the same name in another module.
@@ -37,9 +41,7 @@ class Database:
     Property file_per_database returns False because all 'key:value' sets,
     and the associated inverted list indicies, are held in one file by
     default.  Berkeley DB is the known example where it is reasonable to
-    return True: override the class attribute, and the archive() and
-    delete_archive() methods in .archivedu.Archivedu class, if this case
-    is wanted.
+    return True: override the class attribute, if this case is wanted.
 
     """
 
@@ -133,7 +135,6 @@ class Database:
             self.delete(dbset, oldkey, instance.srvalue)
             key = self.put(dbset, newkey, instance.newrecord.srvalue)
             if key is not None:
-
                 # put was append to record number database and
                 # returned the new primary key. Adjust record key
                 # for secondary updates.
@@ -223,7 +224,6 @@ class Database:
         putkey = instance.key.pack()
         instance.set_packed_value_and_indexes()
         if putkey is None:
-
             # reuse record number if possible
             putkey = self.get_lowest_freed_record_number(dbset)
             if putkey is not None:
@@ -231,7 +231,6 @@ class Database:
 
         key = self.put(dbset, putkey, instance.srvalue)
         if key is not None:
-
             # put was append to record number database and
             # returned the new primary key. Adjust record key
             # for secondary updates.
@@ -379,6 +378,72 @@ class Database:
         """Do nothing, present for compatibility with DPT."""
         del kwargs
 
+    def get_database_table_sizes(self, **kwargs):
+        """Return empty dict, present for compatibility with DPT."""
+        del kwargs
+        return {}
+
+    def close_datasourcecursor_recordset(self, datasourcecursor):
+        """Do nothing.  Override if necessary.
+
+        Present for compatibility with DPT database engine.
+        """
+        del datasourcecursor
+
+    def set_datasourcecursor_recordset(self, datasourcecursor, recordset):
+        """Validate and set recordset as datasourcecursor's recordset.
+
+        The recordset and datasourcecursor must be associated with the same
+        database identity and be the identity of self.
+
+        This method exists because in DPT these checks are implied when the
+        recordset and cursor are created, and both objects are details of
+        the implementation.
+        """
+        if datasourcecursor.dbhome is not self:
+            raise DataSourceCursorError("DataSource is not for this database")
+        if datasourcecursor.recordset:
+            if (
+                datasourcecursor.recordset.dbidentity
+                == recordset.recordset.dbidentity
+            ):
+                datasourcecursor.recordset.recordset.close()
+                datasourcecursor.recordset = recordset
+            else:
+                raise DataSourceCursorError(
+                    "New and existing Recordsets are for different databases"
+                )
+        elif datasourcecursor.dbidentity == recordset.recordset.dbidentity:
+            datasourcecursor.recordset = recordset
+        else:
+            raise DataSourceCursorError(
+                "New Recordset and DataSource are for different databases"
+            )
+
+    def get_datasourcecursor_recordset_cursor(self, dsc):
+        """Create and return cursor on dsc's recordset.
+
+        dsc not datasourcecursor to shorten argument name.
+
+        """
+        if dsc.dbhome is not self:
+            raise DataSourceCursorError("DataSource is not for this database")
+        if dsc.recordset:
+            if dsc.dbidentity == dsc.recordset.recordset.dbidentity:
+                cursor = dsc.recordset.dbhome.create_recordset_cursor(
+                    dsc.recordset.recordset
+                )
+            else:
+                raise DataSourceCursorError(
+                    "Recordset and DataSource are for different databases"
+                )
+        else:
+            dsc.recordset = dsc.dbhome.recordlist_nil(dsc.dbset)
+            cursor = dsc.recordset.dbhome.create_recordset_cursor(
+                dsc.recordset.recordset
+            )
+        return cursor
+
 
 class ExistenceBitmapControl:
     """Base class for managing existence bitmap of file in database.

{solentware-base-5.1 → solentware-base-5.2}/solentware_base/core/_databasedu.py

@@ -13,6 +13,7 @@ from . import _database
 from .segmentsize import SegmentSize
 from .constants import SECONDARY
 from .bytebit import Bitarray
+from . import merge
 
 
 class DatabaseduError(Exception):
@@ -22,25 +23,6 @@ class DatabaseduError(Exception):
 class Database(_database.Database):
     """Provide deferred update versions of the record update methods."""
 
-    # Deferred updates are done inside transactions if possible, despite
-    # it being slower, so backup should not be necessary in most cases.
-    # Where deferred update within transactions is not available, or is
-    # wanted perhaps, a subclass of _databasedu.Database should override
-    # this attribute to 'True'.
-    # This and it's property, and methods archive and delete_archive are
-    # duplicated in the dptdu_database.Database hierarchy: enough for a
-    # shared superclass for default backup stuff.
-    _take_backup_before_deferred_update = False
-
-    @property
-    def take_backup_before_deferred_update(self):
-        """Return True if temporary backups should protect deferred update.
-
-        It is expected the archive and delete_archive methods will do this.
-
-        """
-        return self._take_backup_before_deferred_update
-
     def put_instance(self, dbset, instance):
         """Put new instance on database dbset.
 
@@ -66,7 +48,7 @@ class Database(_database.Database):
         instance.srkey = self.encode_record_number(putkey)
         srindex = instance.srindex
         segment, record_number = divmod(putkey, SegmentSize.db_segment_size)
-        self.defer_add_record_to_ebm(dbset, segment, record_number)
+        self._defer_add_record_to_ebm(dbset, segment, record_number)
         pcb = instance.putcallbacks
         for secondary in srindex:
             if secondary not in self.specification[dbset][SECONDARY]:
@@ -74,12 +56,12 @@ class Database(_database.Database):
                     pcb[secondary](instance, srindex[secondary])
                 continue
             for j in srindex[secondary]:
-                self.defer_add_record_to_field_value(
+                self._defer_add_record_to_field_value(
                     dbset, secondary, j, segment, record_number
                 )
 
         if record_number in self.deferred_update_points:
-            self.write_existence_bit_map(dbset, segment)
+            self._write_existence_bit_map(dbset, segment)
             for secondary in self.specification[dbset][SECONDARY]:
                 self.sort_and_write(dbset, secondary, segment)
             if record_number == max(self.deferred_update_points):
@@ -88,7 +70,45 @@ class Database(_database.Database):
                 self.first_chunk[dbset] = False
                 self.high_segment[dbset] = segment
 
-    def defer_add_record_to_ebm(self, file, segment, record_number):
+    def index_instance(self, dbset, instance):
+        """Apply instance index values on database dbset.
+
+        This method assumes all primary databases are integer primary key,
+        and there is enough memory to do a segment at a time.
+
+        """
+        putkey = instance.key.pack()
+        if putkey is None:
+            # Apply index values without record number is not allowed.
+            raise DatabaseduError(
+                "Cannot apply index values without a record number"
+            )
+        instance.set_packed_value_and_indexes()
+        instance.srkey = self.encode_record_number(putkey)
+        srindex = instance.srindex
+        segment, record_number = divmod(putkey, SegmentSize.db_segment_size)
+        pcb = instance.putcallbacks
+        for secondary in srindex:
+            if secondary not in self.specification[dbset][SECONDARY]:
+                if secondary in pcb:
+                    pcb[secondary](instance, srindex[secondary])
+                continue
+            for j in srindex[secondary]:
+                self._defer_add_record_to_field_value(
+                    dbset, secondary, j, segment, record_number
+                )
+
+        if record_number in self.deferred_update_points:
+            for secondary in srindex:
+                assert secondary in self.specification[dbset][SECONDARY]
+                self.sort_and_write(dbset, secondary, segment)
+            if record_number == max(self.deferred_update_points):
+                self.first_chunk[dbset] = True
+            elif record_number == min(self.deferred_update_points):
+                self.first_chunk[dbset] = False
+                self.high_segment[dbset] = segment
+
+    def _defer_add_record_to_ebm(self, file, segment, record_number):
         """Add bit to existence bit map for new record and defer update."""
         assert file in self.specification
         try:
@@ -110,7 +130,7 @@ class Database(_database.Database):
                 self.existence_bit_maps[file] = {}
             self.existence_bit_maps[file][segment] = ebm
 
-    def defer_add_record_to_field_value(
+    def _defer_add_record_to_field_value(
         self, file, field, key, segment, record_number
     ):
         """Add record_number to cached segment for key."""
@@ -126,7 +146,6 @@ class Database(_database.Database):
         if values is None:
             value_segments[key] = [record_number]
         elif isinstance(values, list):
-
             # A (value, record_number) can be given many times.
             # Ensure a record_number appears in the list once only.
             if values[-1] != record_number:
@@ -159,7 +178,6 @@ class Database(_database.Database):
         for k in segvalues:
             value = segvalues[k]
             if isinstance(value, list):
-
                 # A single record is presented as an integer: the
                 # database engine will decide the transformation.
                 if len(value) == 1:
@@ -191,3 +209,64 @@ class Database(_database.Database):
         Actions are specific to a database engine.
 
         """
+
+    def encode_number_for_sequential_file_dump(self, number, bytes_):
+        """Return big-endian encoding of number in bytes_ bytes."""
+        return number.to_bytes(bytes_, byteorder="big")
+
+    def encode_segment_for_sequential_file_dump(self, record_numbers):
+        """Return encoding of record numbers appropriate to record count."""
+        if len(record_numbers) > SegmentSize.db_upper_conversion_limit:
+            seg = SegmentSize.empty_bitarray.copy()
+            for bit in record_numbers:
+                seg[bit] = True
+            return seg.tobytes()
+        int_to_bytes = self._int_to_bytes
+        return b"".join([int_to_bytes[n] for n in record_numbers])
+
+    def set_int_to_bytes_lookup(self, lookup=True):
+        """Create lookup to convert integers to bytestring big-endian.
+
+        Set lookup table to None if bool(lookup) is False.
+
+        """
+        if lookup:
+            self._int_to_bytes = [
+                n.to_bytes(2, byteorder="big")
+                for n in range(SegmentSize.db_segment_size)
+            ]
+        else:
+            self._int_to_bytes = None
+
+    def merge_import(self, index_directory, file, field, commit_limit):
+        """Yield count of sorted items written to an index at intervals.
+
+        Sorted items from files in index_directory are written to index
+        field in file, and the count of items added is yielded every
+        commit_limit items.
+
+        """
+        writer = self.merge_writer(file, field)
+        merger = merge.Merge(index_directory)
+        commit_count = None
+        for commit_count, item in enumerate(merger.sorter()):
+            if not commit_count % commit_limit:
+                if commit_count:
+                    writer.close_cursor()
+                    self.commit()
+                    self.deferred_update_housekeeping()
+                    yield commit_count
+                    self.start_transaction()
+                    writer.make_new_cursor()
+            writer.write(item)
+        writer.close_cursor()
+        if commit_count is not None:
+            self.commit()
+            self.deferred_update_housekeeping()
+            self.start_transaction()
+
+    def next_sorted_item(self, index_directory):
+        """Yield sorted items from files in index_directory."""
+        merger = merge.Merge(index_directory)
+        for item in merger.sorter():
+            yield item