ic-python-db 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

ic_python_db/__init__.py

@@ -22,7 +22,7 @@ from .schema import SchemaIncompatibleError, build_schema, diff_schemas, schema_
 from .storage import MemoryStorage, Storage
 from .system_time import SystemTime
 
-__version__ = "0.13.0"
+__version__ = "0.14.0"
 __all__ = [
     "Database",
     "Entity",

ic_python_db/db_engine.py

@@ -30,6 +30,11 @@ _PQ_HEAD = "_pq:head"
 _PQ_TAIL = "_pq:tail"
 # Entries of deleted sets each index write removes on the side
 _PURGE_PER_WRITE = 20
+# Sorted indexes (see "Sorted indexes" below): each field's value is encoded
+# in _SORT_WIDTH decimal digits, an entity id in _SORT_ID_WIDTH characters
+_SORT_WIDTH = 20
+_SORT_ID_WIDTH = 13
+_INVERT = str.maketrans("0123456789", "9876543210")
 
 
 def _is_sequential_id(id_str: str) -> bool:
@@ -855,6 +860,81 @@ class Database:
         """Remove an entity ID from the index entry for ``value``."""
         self._ix_remove(self._fi_key(type_name, field_name, value), entity_id)
 
+    # ── Sorted indexes ─────────────────────────────────────────────────────────
+    #
+    # A sorted index keeps one entry per entity whose key starts with the
+    # entity's sort values, so the storage's own key order is the sort order
+    # and a "top N" is one range read:
+    #
+    #   _si:{Type}:{fields}:a|{values}{id}  -> entity id   lowest first
+    #   _si:{Type}:{fields}:d|{values}{id}  -> entity id   highest first
+    #   _sb:{Type}:{fields}:{entity id}     -> {values}    what is indexed now
+    #
+    # {values} is each field's value as _SORT_WIDTH digits (Entity encodes
+    # them), {id} the id in _SORT_ID_WIDTH characters. Every key of one index
+    # and direction has the same length, which keeps them contiguous in key
+    # order (storage_sort_key). The "d" entry stores every digit inverted, so
+    # ascending key order is descending value order; ties go to the higher id.
+    #
+    # The back pointer (_sb) lets an update or a rebuild remove the stale
+    # entry without knowing the old value, so an index survives data
+    # migrations and type changes.
+
+    def _si_prefix(self, type_name: str, index: str, descending: bool) -> str:
+        return f"_si:{type_name}:{index}:{'d' if descending else 'a'}|"
+
+    @staticmethod
+    def _si_id(entity_id: str, descending: bool) -> str:
+        if _is_sequential_id(entity_id) and len(entity_id) < _SORT_ID_WIDTH:
+            digits = entity_id.zfill(_SORT_ID_WIDTH - 1)
+            return "n" + (digits.translate(_INVERT) if descending else digits)
+        digest = hashlib.sha256(entity_id.encode("utf-8")).hexdigest()
+        return "h" + digest[: _SORT_ID_WIDTH - 1]
+
+    def sorted_index_set(
+        self, type_name: str, index: str, entity_id: str, values: Optional[str]
+    ) -> None:
+        """Index ``entity_id`` under ``values`` (encoded sort values), or drop
+        it when ``values`` is None. A no-op when nothing changed."""
+        back = f"_sb:{type_name}:{index}:{entity_id}"
+        old = self._db_storage.get(back)
+        if old == values:
+            return
+        storage = self._db_storage
+        for descending in (False, True):
+            prefix = self._si_prefix(type_name, index, descending)
+            ident = self._si_id(entity_id, descending)
+            if old is not None:
+                key = prefix + (old.translate(_INVERT) if descending else old) + ident
+                if storage.get(key) is not None:
+                    storage.remove(key)
+            if values is not None:
+                key = prefix + (values.translate(_INVERT) if descending else values)
+                storage.insert(key + ident, entity_id)
+        if values is None:
+            storage.remove(back)
+        else:
+            storage.insert(back, values)
+
+    def sorted_index_scan(
+        self,
+        type_name: str,
+        index: str,
+        fields: int,
+        descending: bool,
+        start: Optional[str],
+        limit: int,
+    ) -> List[Tuple[str, str]]:
+        """Up to ``limit`` (key, entity id) pairs of an index in sort order,
+        from key ``start`` (inclusive; None for the first)."""
+        prefix = self._si_prefix(type_name, index, descending)
+        width = _SORT_WIDTH * fields + _SORT_ID_WIDTH
+        if start is None:
+            start = prefix + "0" * width
+        elif not start.startswith(prefix) or len(start) != len(prefix) + width:
+            raise ValueError("cursor does not belong to this sorted index")
+        return self._scan(start, prefix + "~" * width, limit)
+
     # Reverse counters (unidirectional relations)
     # Key format: "_rc:{parent_type}:{parent_id}:{relation_name}"
     # Value: integer count of related entities.

ic_python_db/entity.py

@@ -1,6 +1,6 @@
 """Enhanced entity implementation with support for mixins and entity types."""
 
-from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar
 
 from ic_python_logging import get_logger
 
@@ -12,6 +12,44 @@ logger = get_logger(__name__)
 
 T = TypeVar("T", bound="Entity")
 
+_I64_MIN, _I64_LIMIT = -(1 << 63), 1 << 63
+_INF = float("inf")
+
+
+def _float_bits(value: float) -> int:
+    """The IEEE 754 bit pattern of abs(value), read from float.hex().
+
+    The canister's ``struct`` module is a stub without pack/unpack.
+    """
+    mantissa, exponent = float.hex(abs(value))[2:].split("p")
+    lead, fraction = mantissa.split(".")
+    fraction_bits = int(fraction.ljust(13, "0"), 16)
+    if lead == "0":
+        return fraction_bits  # zero or subnormal: biased exponent 0
+    return ((int(exponent) + 1023) << 52) | fraction_bits
+
+
+def _sort_encode(value: Any) -> str:
+    """A value as 20 decimal digits whose string order is the value's order.
+
+    Integers are offset into 0..2**64-1; finite floats use their IEEE 754
+    bits around 2**63, so that negative numbers come first.
+    """
+    if isinstance(value, bool):
+        value = int(value)
+    if isinstance(value, int):
+        if not _I64_MIN <= value < _I64_LIMIT:
+            raise ValueError(f"sorted Integer values must fit in 64 bits: {value}")
+        unsigned = value - _I64_MIN
+    elif isinstance(value, float):
+        if value != value or value in (_INF, -_INF):
+            raise ValueError(f"sorted Float values must be finite: {value}")
+        bits = _float_bits(value)
+        unsigned = (1 << 63) - bits if value < 0 else (1 << 63) + bits
+    else:
+        raise TypeError(f"sorted fields hold numbers, got {type(value).__name__}")
+    return str(unsigned).zfill(20)
+
 
 class Entity:
     """Base class for database entities with enhanced features.
@@ -439,6 +477,7 @@ class Entity:
                 f"Version mismatch for {type_name}@{entity_id}: "
                 f"stored={stored_version}, current={current_version}"
             )
+            stored = dict(data)  # migrate() may change data in place
             # Apply custom migration first (developer logic takes priority)
             data = cls.migrate(data, stored_version, current_version)
             # Auto-inject defaults for new fields not handled by migrate()
@@ -451,12 +490,25 @@ class Entity:
         # Create instance first
         entity = cls(**data, _loaded=True)
 
-        # If migration was applied, persist the migrated data
+        # If migration was applied, persist the migrated data and move its
+        # index entries along with any value migrate() changed
         if stored_version != current_version:
             entity._save()
+            entity._reindex_after_migration(stored)
 
         return entity
 
+    def _reindex_after_migration(self, stored: dict) -> None:
+        db = self.db()
+        for field in self._indexed_properties():
+            old, new = stored.get(field), getattr(self, field, None)
+            if old != new:
+                if old is not None:
+                    db.field_index_remove(self._type, field, str(old), self._id)
+                if new is not None:
+                    db.field_index_add(self._type, field, str(new), self._id)
+        self._sync_sorted()
+
     @classmethod
     def _require_unpaged(cls, method: str) -> None:
         """Raise TooLargeError before ``method`` loads more than MAX_UNPAGED entities."""
@@ -723,6 +775,168 @@ class Entity:
         next_from_id = int(rows[count][0]) if len(rows) > count else None
         return entities, next_from_id
 
+    @classmethod
+    def _sorted_indexes(cls) -> Dict[str, Tuple[str, ...]]:
+        """{index name: fields} for this type's sorted indexes.
+
+        A property declared ``sorted=True`` gets its own index; a class's
+        ``__sorted_indexes__`` adds indexes over several fields, compared in
+        order (the second breaks ties in the first, and so on).
+        """
+        cached = cls.__dict__.get("_sorted_cache")
+        if cached is not None:
+            return cached
+        from .properties import Property
+
+        props: Dict[str, Any] = {}
+        for klass in reversed(cls.__mro__):
+            for attr_name, attr_value in klass.__dict__.items():
+                if not attr_name.startswith("_") and isinstance(attr_value, Property):
+                    props[attr_name] = attr_value
+        indexes: Dict[str, Tuple[str, ...]] = {
+            name: (name,) for name, prop in props.items() if prop.sorted
+        }
+        for spec in getattr(cls, "__sorted_indexes__", ()) or ():
+            fields = (spec,) if isinstance(spec, str) else tuple(spec)
+            for field in fields:
+                prop = props.get(field)
+                if prop is None or prop.type not in (int, float):
+                    raise TypeError(
+                        f"{cls.__name__}.__sorted_indexes__: {field} is not an "
+                        f"Integer or Float property"
+                    )
+            indexes[",".join(fields)] = fields
+        cls._sorted_cache = indexes
+        return indexes
+
+    def _sync_sorted(
+        self, field: Optional[str] = None, index: Optional[str] = None
+    ) -> None:
+        """Bring this entity's sorted index entries up to date: the indexes
+        that contain ``field``, only ``index``, or all of them."""
+        if self._id is None:
+            return
+        db = self.db()
+        for name, fields in self._sorted_indexes().items():
+            if (field is not None and field not in fields) or (
+                index is not None and name != index
+            ):
+                continue
+            values = [getattr(self, f, None) for f in fields]
+            encoded = (
+                None
+                if any(v is None for v in values)
+                else "".join(_sort_encode(v) for v in values)
+            )
+            db.sorted_index_set(self._type, name, self._id, encoded)
+
+    @classmethod
+    def _sorted_index_name(cls, fields) -> Tuple[str, Tuple[str, ...]]:
+        key = (fields,) if isinstance(fields, str) else tuple(fields)
+        name = ",".join(key)
+        if name not in cls._sorted_indexes():
+            raise ValueError(
+                f"{cls.__name__} has no sorted index on {name}: declare the "
+                f"property sorted=True, or list the fields in __sorted_indexes__"
+            )
+        return name, key
+
+    @classmethod
+    def top_by(
+        cls: Type[T],
+        fields,
+        count: int = 20,
+        cursor: Optional[str] = None,
+        descending: bool = True,
+        where: Optional[Callable[[Any], bool]] = None,
+        scan_limit: int = 200,
+    ) -> Tuple[List[T], Optional[str]]:
+        """One page of entities in the order of a sorted index.
+
+        ``fields`` is a field declared ``sorted=True``, or a tuple listed in
+        ``__sorted_indexes__``. Highest first unless ``descending=False``;
+        ties go to the higher id. Entities with None in any of the fields are
+        not indexed.
+
+        Returns (entities, cursor); pass the cursor back for the next page,
+        None once the index is exhausted. ``where`` drops entities that fail
+        it. At most ``scan_limit`` entries are read per call, so with a
+        selective ``where`` a page can come back short, or empty, with a
+        cursor: call again. Cost is bounded by ``scan_limit`` loads, however
+        many entities exist::
+
+            top, cursor = Listing.top_by("installs", count=20)
+            top, cursor = Listing.top_by(("installs", "likes"), count=20)
+        """
+        if count < 1:
+            raise ValueError("count must be at least 1")
+        if scan_limit < 1:
+            raise ValueError("scan_limit must be at least 1")
+        name, key = cls._sorted_index_name(fields)
+        db = cls.db()
+        type_name = cls.get_full_type_name()
+        found: List[T] = []
+        read = 0
+        start = cursor
+        chunk = count  # with `where`, doubled each round: rows read track rows kept
+        while len(found) < count and read < scan_limit:
+            want = min(
+                count - len(found) if where is None else chunk, scan_limit - read, 100
+            )
+            chunk *= 2
+            rows = db.sorted_index_scan(
+                type_name, name, len(key), descending, start, want
+            )
+            for row_key, entity_id in rows:
+                read += 1
+                # The next same-length key: where a later page starts
+                start = row_key[:-1] + chr(ord(row_key[-1]) + 1)
+                entity = cls.load(entity_id)
+                if entity is not None and (where is None or where(entity)):
+                    found.append(entity)
+                    if len(found) >= count:
+                        break
+            else:
+                if len(rows) < want:
+                    return found, None  # index exhausted
+        if start is None or not db.sorted_index_scan(
+            type_name, name, len(key), descending, start, 1
+        ):
+            return found, None
+        return found, start
+
+    @classmethod
+    def rebuild_sorted_index(
+        cls: Type[T], fields=None, from_id: int = 1, batch: int = 200
+    ) -> Optional[int]:
+        """Index one batch of existing entities in a sorted index (all of
+        them when ``fields`` is None); resumable and idempotent.
+
+        Use it after declaring a field sorted, or after changing a sorted
+        field's type: each entity's stale entry is replaced, whatever it was::
+
+            cursor = 1
+            while cursor is not None:
+                cursor = Listing.rebuild_sorted_index("installs", from_id=cursor)
+
+        Returns the from_id for the next batch, or None when done.
+        """
+        if from_id < 1:
+            raise ValueError("from_id must be at least 1")
+        if batch < 1:
+            raise ValueError("batch must be at least 1")
+        name = None if fields is None else cls._sorted_index_name(fields)[0]
+        max_id = cls.max_id()
+        if max_id == 0 or from_id > max_id:
+            return None
+        entities = cls.load_some(from_id, batch)
+        for entity in entities:
+            entity._sync_sorted(index=name)
+        if len(entities) < batch:
+            return None
+        next_from_id = int(entities[-1]._id) + 1
+        return next_from_id if next_from_id <= max_id else None
+
     @classmethod
     def count(cls: Type[T]) -> int:
         """Get the total count of entities of this type.
@@ -945,6 +1159,9 @@ class Entity:
             if value is not None:
                 db.field_index_remove(self._type, field_name, str(value), self._id)
 
+        for name in self._sorted_indexes():
+            db.sorted_index_set(self._type, name, self._id, None)
+
         db.delete(self._type, self._id)
 
         # Remove from entity registry

ic_python_db/properties.py

@@ -42,6 +42,7 @@ class Property(Generic[T]):
     default: Optional[T]
     validator: Optional[Callable[[T], bool]]
     indexed: bool
+    sorted: bool
 
     def __init__(
         self,
@@ -50,12 +51,14 @@ class Property(Generic[T]):
         default: Optional[T] = None,
         validator: Optional[Callable[[T], bool]] = None,
         indexed: bool = False,
+        sorted: bool = False,
     ):
         self.name = name
         self.type = type
         self.default = default
         self.validator = validator
         self.indexed = indexed
+        self.sorted = sorted
 
     def __set_name__(self, owner: type, name: str) -> None:
         self.name = name
@@ -121,6 +124,9 @@ class Property(Generic[T]):
             if value is not None:
                 db.field_index_add(obj._type, self.name, str(value), obj._id)
 
+        if old_value != value and obj._id is not None and hasattr(obj, "_sync_sorted"):
+            obj._sync_sorted(self.name)
+
         obj._save()
 
 
@@ -155,6 +161,7 @@ class Integer(Property[int]):
         max_value: Optional[int] = None,
         default: Optional[int] = None,
         indexed: bool = False,
+        sorted: bool = False,
     ):
         def validator(value: int) -> bool:
             if min_value is not None and value < min_value:
@@ -164,7 +171,12 @@ class Integer(Property[int]):
             return True
 
         super().__init__(
-            name="", type=int, default=default, validator=validator, indexed=indexed
+            name="",
+            type=int,
+            default=default,
+            validator=validator,
+            indexed=indexed,
+            sorted=sorted,
         )
 
 
@@ -177,6 +189,7 @@ class Float(Property[float]):
         max_value: Optional[float] = None,
         default: Optional[float] = None,
         indexed: bool = False,
+        sorted: bool = False,
     ):
         def validator(value: float) -> bool:
             if min_value is not None and value < min_value:
@@ -186,7 +199,12 @@ class Float(Property[float]):
             return True
 
         super().__init__(
-            name="", type=float, default=default, validator=validator, indexed=indexed
+            name="",
+            type=float,
+            default=default,
+            validator=validator,
+            indexed=indexed,
+            sorted=sorted,
         )
 
 

{ic_python_db-0.13.0.dist-info → ic_python_db-0.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ic-python-db
-Version: 0.13.0
+Version: 0.14.0
 Summary: A lightweight key-value database written in Python, intended for use on the Internet Computer (IC)
 Home-page: https://github.com/smart-social-contracts/ic-python-db
 Author: Smart Social Contracts
@@ -232,6 +232,31 @@ The rebuild is idempotent. The `indexed` flag is not part of the schema descript
 
 Each matching id is its own storage entry, so a value can match any number of entities without reaching the backing store's `max_value_size`.
 
+## Sorted Fields
+
+Declare `sorted=True` on an `Integer` or `Float` property to read entities in its order, highest or lowest first, without loading and sorting them all. `__sorted_indexes__` adds indexes over several fields, where each field breaks ties in the one before:
+
+```python
+class Listing(Entity):
+    name = String()
+    installs = Integer(sorted=True)
+    likes = Integer(sorted=True)
+    updated_at = Float(sorted=True)
+    __sorted_indexes__ = [("installs", "likes")]   # installs, then likes
+
+top, cursor = Listing.top_by("installs", count=20)              # most installed
+top, cursor = Listing.top_by(("installs", "likes"), count=20)   # ties broken by likes
+recent, cursor = Listing.top_by("updated_at", count=20)
+oldest, cursor = Listing.top_by("updated_at", count=20, descending=False)
+more, cursor = Listing.top_by("installs", count=20, cursor=cursor)   # next page; None at the end
+```
+
+The storage map keeps keys in order, so each entity has one index entry per sorted index whose key starts with its values. Writing a sorted field moves that entry (a few storage operations), and a page is one range read plus `count` loads, however many entities exist. Ties go to the higher id; entities with None in a field are left out of indexes on it. Integers must fit in 64 bits.
+
+`where=` drops entities that fail a test. At most `scan_limit` entries (default 200) are read per call, so with a selective filter a page can come back short with a cursor: call again.
+
+Fields made sorted after rows exist, or whose type changed, need a resumable rebuild (`Listing.rebuild_sorted_index("installs", from_id=cursor)`, until it returns None). A `migrate()` that changes a sorted or indexed value moves the entity's entries when it is loaded.
+
 ## Large Relations
 
 `OneToMany` and `ManyToMany` getters return a `RelationView`, not a list. Questions about the relation and single pages never load every member, so they stay cheap however large it grows:

{ic_python_db-0.13.0.dist-info → ic_python_db-0.14.0.dist-info}/RECORD

@@ -1,19 +1,19 @@
-ic_python_db/__init__.py,sha256=amMTKKg0eRKGSc6Hz8O-ZqhZfwOZFwRbfxxB5UQfX5o,1083
+ic_python_db/__init__.py,sha256=6qHPpgma0wqCnnOVrfT_Wp3KEFHy6Q20XFS22_-kh6o,1083
 ic_python_db/_cdk.py,sha256=zTj8HzXUcPz8R73nh5bo6JMkCorp-KSW1hDutzi5LZs,387
 ic_python_db/constants.py,sha256=gs6klyqUxUjE0LFJxXpaya8QEgYeKx4ovP9Mu04rtHk,351
 ic_python_db/context.py,sha256=97F4_EAJm3adOnUfh0SaDisX3nc7F2vFIZwZUsUAy58,1080
-ic_python_db/db_engine.py,sha256=KniUAn9Gzn8G8bhj-8Kz0-Q1xX6dIvtcN3t3Ng-STSc,36792
-ic_python_db/entity.py,sha256=fRUN0gBBN-KtEXOZOh5rxXlskFltfG8kDlwq1fK6Ck0,54408
+ic_python_db/db_engine.py,sha256=v6jjC4wcjBOpXRNjw62gV9RC3Own795joLp00l3GUfM,40661
+ic_python_db/entity.py,sha256=zXru0y1eGRImT1vdIMA-l8fcGvOAWHyFps4XwuyuOlQ,63458
 ic_python_db/errors.py,sha256=q_2oAu2mX4B3eGySS-8xOU_CIoKoPlxKwECkbxaHQ1E,366
 ic_python_db/hooks.py,sha256=B1wV18dAMH-PootSripmMpG9Cr9M3W3lcNEF7QvPrL4,1620
 ic_python_db/mixins.py,sha256=qZmb3ssXYf2NSjHJtgmdsOlkIJdouDjPHdQ9tZmrZi8,1937
-ic_python_db/properties.py,sha256=yo2mPH2OjzYvGM3KKSAaiIGTep_qv2496ZT9-_T34FU,31253
+ic_python_db/properties.py,sha256=FZgh9FFL8epypEo0u9y8zi0HLXz0KYl3nXtY4ILlnQE,31670
 ic_python_db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ic_python_db/schema.py,sha256=h1RAbapExo8pMD5IQvK8ZTVkgyonbPV9qSqliSP0Y14,16249
 ic_python_db/storage.py,sha256=899qyuA3alpAFIPM23A_07vqquUB5ZG5GjjpzlHW1Os,3828
 ic_python_db/system_time.py,sha256=xrx8NfjLXCGsac3hUpuvsNwCH91tQ9jfH4N-XFo8iJk,2614
-ic_python_db-0.13.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
-ic_python_db-0.13.0.dist-info/METADATA,sha256=lwTLW2L4sj_aLQ--SxXjAFZ0WBRb7of1UXi7YqCa83I,18319
-ic_python_db-0.13.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-ic_python_db-0.13.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
-ic_python_db-0.13.0.dist-info/RECORD,,
+ic_python_db-0.14.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
+ic_python_db-0.14.0.dist-info/METADATA,sha256=-0meDeN9zfnSvm0mGyRihstBVAkpuQj1AjAHOgZqG_0,20070
+ic_python_db-0.14.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+ic_python_db-0.14.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
+ic_python_db-0.14.0.dist-info/RECORD,,