ic-python-db 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

ic_python_db/__init__.py

@@ -20,7 +20,7 @@ from .schema import SchemaIncompatibleError, build_schema, diff_schemas, schema_
 from .storage import MemoryStorage, Storage
 from .system_time import SystemTime
 
-__version__ = "0.10.0"
+__version__ = "0.12.0"
 __all__ = [
     "Database",
     "Entity",

ic_python_db/db_engine.py

@@ -9,10 +9,41 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from ic_python_logging import get_logger
 
-from .storage import MemoryStorage, Storage
+from .storage import MemoryStorage, Storage, supports_range
 
 logger = get_logger(__name__)
 
+_DIGITS = frozenset("0123456789")
+
+
+def _is_sequential_id(id_str: str) -> bool:
+    """True for canonical sequential ids: ASCII digits, no leading zero."""
+    return bool(id_str) and id_str[0] != "0" and all(c in _DIGITS for c in id_str)
+
+
+def _next_sequential_id(id_str: str, digits: int) -> Optional[int]:
+    """Smallest ``digits``-digit sequential id that sorts after ``id_str``.
+
+    ``id_str`` is a key suffix returned by a range over the ``digits``-length
+    bucket, so it starts with a digit but may be a custom id such as
+    ``"1x"`` or ``"1-a"``. Returns None when no such id exists in the bucket.
+    """
+    i = 0
+    while i < len(id_str) and id_str[i] in _DIGITS:
+        i += 1
+    numeric_prefix, rest = id_str[:i], id_str[i:]
+    if not rest:
+        candidate = str(int(numeric_prefix) + 1)
+    elif rest[0] > "9":
+        # "1x" > every "1<digit>": jump to the next prefix, zero-filled
+        candidate = str(int(numeric_prefix) + 1) + "0" * len(rest)
+    else:
+        # "1-a" < "10": the zero-filled prefix is the next candidate
+        candidate = numeric_prefix + "0" * len(rest)
+    if len(candidate) > digits:
+        return None
+    return max(int(candidate), 10 ** (digits - 1))
+
 
 class Database:
     """Main database class providing high-level operations"""
@@ -181,6 +212,102 @@ class Database:
             return json.loads(data)
         return None
 
+    def max_id(self, type_name: str) -> int:
+        """Highest sequential id ever issued for ``type_name`` (0 if none)."""
+        raw = self.load("_system", f"{type_name}_id")
+        return int(raw) if raw else 0
+
+    def load_page(
+        self,
+        type_name: str,
+        from_id: int,
+        count: int,
+        max_id: Optional[int] = None,
+    ) -> List[Tuple[str, dict]]:
+        """Persisted entities of ``type_name`` with sequential id >= ``from_id``.
+
+        Returns up to ``count`` ``(id, data)`` pairs in ascending numeric id
+        order, skipping ids that were deleted. Only canonical sequential ids
+        (``"1"``, ``"42"``; no leading zeros, no custom string ids) are
+        returned, matching what probing ``1..max_id`` would find.
+
+        When the storage backend supports ``range`` (Basilisk's
+        ``StableBTreeMap``, ``MemoryStorage``) a page costs one range call per
+        digit-count bucket touched (usually one), independent of how many ids
+        were deleted. Otherwise it falls back to one ``get`` per id.
+        """
+        if from_id < 1:
+            raise ValueError("from_id must be at least 1")
+        if count < 1:
+            raise ValueError("count must be at least 1")
+        if max_id is None:
+            max_id = self.max_id(type_name)
+        if max_id < from_id:
+            return []
+        if supports_range(self._db_storage):
+            return self._load_page_range(type_name, from_id, count, max_id)
+        return self._load_page_probe(type_name, from_id, count, max_id)
+
+    def _load_page_probe(
+        self, type_name: str, from_id: int, count: int, max_id: int
+    ) -> List[Tuple[str, dict]]:
+        results: List[Tuple[str, dict]] = []
+        entity_id = from_id
+        while len(results) < count and entity_id <= max_id:
+            raw = self._db_storage.get(f"{type_name}@{entity_id}")
+            if raw:
+                results.append((str(entity_id), json.loads(raw)))
+            entity_id += 1
+        return results
+
+    def _load_page_range(
+        self, type_name: str, from_id: int, count: int, max_id: int
+    ) -> List[Tuple[str, dict]]:
+        # Keys are "Type@<id>" with unpadded decimal ids. The backend orders
+        # str keys by byte length first, then bytewise (see storage_sort_key),
+        # so all ids with the same digit count form one contiguous, numerically
+        # ordered run. Walk one run ("bucket") per digit count, from the digit
+        # count of from_id up to that of max_id.
+        prefix = f"{type_name}@"
+        results: List[Tuple[str, dict]] = []
+        digits = len(str(from_id))
+        max_digits = len(str(max_id))
+        while digits <= max_digits and len(results) < count:
+            lo_id = max(from_id, 10 ** (digits - 1))
+            if digits == max_digits:
+                # Exclusive upper bound right after max_id (same digit count),
+                # or the whole bucket if max_id is the largest d-digit number.
+                hi_id = max_id + 1
+                end = (
+                    f"{prefix}{hi_id}"
+                    if len(str(hi_id)) == digits
+                    else f"{prefix}{':' * digits}"
+                )
+            else:
+                # ':' is the character after '9': every same-length key of
+                # the form "Type@<digits>" sorts before "Type@:::".
+                end = f"{prefix}{':' * digits}"
+
+            start = f"{prefix}{lo_id}"
+            plen = len(prefix)
+            while len(results) < count:
+                need = count - len(results)
+                page = self._db_storage.range(start, end, need)
+                for key, raw in page:
+                    id_str = key[plen:]
+                    if _is_sequential_id(id_str):
+                        results.append((id_str, json.loads(raw)))
+                if len(page) < need:
+                    break  # bucket exhausted
+                # Page was full but may have contained skipped custom ids:
+                # continue from the next possible sequential id in this bucket.
+                next_id = _next_sequential_id(page[-1][0][plen:], digits)
+                if next_id is None:
+                    break
+                start = f"{prefix}{next_id}"
+            digits += 1
+        return results
+
     def delete(self, type_name: str, entity_id: str) -> None:
         """Delete the data associated with the key
 
@@ -426,6 +553,56 @@ class Database:
         if self._db_storage.get(key) is not None:
             self._db_storage.remove(key)
 
+    # ── Field Index API (issue #11) ────────────────────────────────────────────
+    #
+    # Secondary indexes for scalar properties declared with indexed=True.
+    # Allow equality lookups (Entity.find_by) without scanning all entities.
+    #
+    # Key format: _fi:{Type}:{field}:{value}
+    # Value: JSON array of entity IDs, e.g. ["1", "2", "3"]
+
+    def _fi_key(self, type_name: str, field_name: str, value: str) -> str:
+        return f"_fi:{type_name}:{field_name}:{value}"
+
+    def field_index_get(self, type_name: str, field_name: str, value: str) -> List[str]:
+        """Read the ID list for entities whose ``field_name`` equals ``value``.
+
+        Returns:
+            List of entity IDs, or empty list if no index entry exists.
+        """
+        key = self._fi_key(type_name, field_name, value)
+        raw = self._db_storage.get(key)
+        if raw:
+            return json.loads(raw)
+        return []
+
+    def field_index_add(
+        self, type_name: str, field_name: str, value: str, entity_id: str
+    ) -> None:
+        """Add an entity ID to the index entry for ``value``."""
+        key = self._fi_key(type_name, field_name, value)
+        raw = self._db_storage.get(key)
+        ids = json.loads(raw) if raw else []
+        if entity_id not in ids:
+            ids.append(entity_id)
+            self._db_storage.insert(key, json.dumps(ids))
+
+    def field_index_remove(
+        self, type_name: str, field_name: str, value: str, entity_id: str
+    ) -> None:
+        """Remove an entity ID from the index entry for ``value``."""
+        key = self._fi_key(type_name, field_name, value)
+        raw = self._db_storage.get(key)
+        if not raw:
+            return
+        ids = json.loads(raw)
+        if entity_id in ids:
+            ids.remove(entity_id)
+            if ids:
+                self._db_storage.insert(key, json.dumps(ids))
+            else:
+                self._db_storage.remove(key)
+
     # Reverse counters (unidirectional relations)
     # Key format: "_rc:{parent_type}:{parent_id}:{relation_name}"
     # Value: integer count of related entities.

ic_python_db/entity.py

@@ -1,6 +1,6 @@
 """Enhanced entity implementation with support for mixins and entity types."""
 
-from typing import Any, Dict, List, Optional, Set, Type, TypeVar
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar
 
 from ic_python_logging import get_logger
 
@@ -176,8 +176,11 @@ class Entity:
             next_id = str(int(current_id) + 1)
             self._id = next_id
             db.save("_system", f"{type_name}_id", self._id)
-        else:
-            # Update max_id if custom ID is higher than current max
+        elif not self._loaded:
+            # Update max_id if custom ID is higher than current max.
+            # Skipped when hydrating persisted state: that id already went
+            # through this check when it was first created, so re-reading the
+            # counter would only add one storage read per loaded entity.
             db = self.db()
             type_name = self._type
             current_max_id = db.load("_system", f"{type_name}_id")
@@ -198,11 +201,17 @@ class Entity:
         self.db().register_entity(self)
 
         self._do_not_save = True
+        # True only while reconstructing persisted state in this constructor:
+        # property descriptors skip hooks, validation, and index maintenance
+        # (indexes are already intact). Explicit updates later — including
+        # deserialize() upserts — run the full descriptor path.
+        self._hydrating = self._loaded
         # Set additional attributes
         for k, v in kwargs.items():
             if not k.startswith("_"):
                 setattr(self, k, v)
         self._do_not_save = False
+        self._hydrating = False
 
         if not self._loaded:
             self._save()
@@ -412,6 +421,16 @@ class Entity:
         if not data:
             return None
 
+        return cls._hydrate(entity_id, data)
+
+    @classmethod
+    def _hydrate(cls: Type[T], entity_id: str, data: dict) -> T:
+        """Build an instance from its persisted dict, migrating if needed.
+
+        Shared by ``load`` (single ``get``) and ``load_some`` (range page).
+        Callers must have checked the identity registry first.
+        """
+        type_name = cls.get_full_type_name()
         stored_version = data.get("__version__", 1)
         current_version = cls.__version__
 
@@ -454,6 +473,140 @@ class Entity:
             if all(getattr(e, k, None) == v for k, v in d.items())
         ]
 
+    @classmethod
+    def _indexed_properties(cls) -> Dict[str, Any]:
+        """Return {name: Property} for all properties declared with indexed=True."""
+        from .properties import Property
+
+        result: Dict[str, Any] = {}
+        for klass in reversed(cls.__mro__):
+            for attr_name, attr_value in klass.__dict__.items():
+                if attr_name.startswith("_"):
+                    continue
+                if isinstance(attr_value, Property) and attr_value.indexed:
+                    result[attr_name] = attr_value
+        return result
+
+    @classmethod
+    def _require_indexed_field(cls, field: str) -> None:
+        if field not in cls._indexed_properties():
+            raise ValueError(
+                f"{cls.__name__}.{field} is not an indexed property "
+                f"(declare it with indexed=True)"
+            )
+
+    @classmethod
+    def find_by(
+        cls: Type[T],
+        field: str,
+        value: Any,
+        from_id: int = 1,
+        count: int = 50,
+    ) -> Tuple[List[T], Optional[int]]:
+        """Equality lookup on an indexed field with cursor pagination (issue #11).
+
+        IDs come from the persistent field index, so cost is proportional to
+        the number of matches, not to max_id. Results are ordered by ascending
+        entity ID, compatible with load_some-style cursors.
+
+        Args:
+            field: Name of a property declared with indexed=True
+            value: Value to match (compared via its string form, like the index)
+            from_id: Only return entities with _id >= from_id
+            count: Maximum number of entities to return
+
+        Returns:
+            (entities, next_from_id) — next_from_id is None when no more
+            matches remain, otherwise pass it back as from_id for the next page.
+
+        Raises:
+            ValueError: If the field is not declared with indexed=True
+        """
+        cls._require_indexed_field(field)
+        if from_id < 1:
+            raise ValueError("from_id must be at least 1")
+        if count < 1:
+            raise ValueError("count must be at least 1")
+
+        ids = cls.db().field_index_get(cls.get_full_type_name(), field, str(value))
+        matching = sorted((int(i) for i in ids if i.isdigit() and int(i) >= from_id))
+
+        entities: List[T] = []
+        last_id = None
+        for entity_id in matching:
+            if len(entities) >= count:
+                break
+            entity = cls.load(str(entity_id))
+            if entity is not None:
+                entities.append(entity)
+            last_id = entity_id
+
+        has_more = last_id is not None and any(i > last_id for i in matching)
+        next_from_id = (last_id + 1) if has_more else None
+        return entities, next_from_id
+
+    @classmethod
+    def count_by(cls: Type[T], field: str, value: Any) -> int:
+        """Number of entities whose indexed ``field`` equals ``value``.
+
+        Raises:
+            ValueError: If the field is not declared with indexed=True
+        """
+        cls._require_indexed_field(field)
+        return len(
+            cls.db().field_index_get(cls.get_full_type_name(), field, str(value))
+        )
+
+    @classmethod
+    def rebuild_field_index(
+        cls: Type[T],
+        field: str,
+        from_id: int = 1,
+        batch: int = 200,
+    ) -> Optional[int]:
+        """Index one batch of pre-existing entities; resumable (issue #11).
+
+        Entities created before a field was declared indexed=True are missing
+        from its index. This walks the ID range in bounded batches so callers
+        on the IC can spread the work across multiple update calls or timer
+        ticks without hitting per-message instruction limits:
+
+            cursor = 1
+            while cursor is not None:
+                cursor = Proposal.rebuild_field_index("org_scope", from_id=cursor)
+
+        Idempotent — re-indexing an already-indexed entity is a no-op.
+
+        Returns:
+            The from_id for the next batch, or None when the rebuild is done.
+
+        Raises:
+            ValueError: If the field is not declared with indexed=True
+        """
+        cls._require_indexed_field(field)
+        if from_id < 1:
+            raise ValueError("from_id must be at least 1")
+        if batch < 1:
+            raise ValueError("batch must be at least 1")
+
+        max_id = cls.max_id()
+        if max_id == 0 or from_id > max_id:
+            return None
+
+        db = cls.db()
+        type_name = cls.get_full_type_name()
+        # One range read per batch of *live* entities (deleted ids cost nothing)
+        entities = cls.load_some(from_id, batch)
+        for entity in entities:
+            value = getattr(entity, field, None)
+            if value is not None:
+                db.field_index_add(type_name, field, str(value), entity._id)
+
+        if len(entities) < batch:
+            return None
+        next_from_id = int(entities[-1]._id) + 1
+        return next_from_id if next_from_id <= max_id else None
+
     @classmethod
     def instances(cls: Type[T]) -> List[T]:
         """Get all instances of this entity type, including subclass instances.
@@ -534,11 +687,7 @@ class Entity:
         Returns:
             int: Maximum entity ID
         """
-        type_name = cls.get_full_type_name()
-        db = cls.db()
-        max_id_key = f"{type_name}_id"
-        max_id = db.load("_system", max_id_key)
-        return int(max_id) if max_id else 0
+        return cls.db().max_id(cls.get_full_type_name())
 
     @classmethod
     def load_some(
@@ -556,29 +705,42 @@ class Entity:
             List[T]: List of entities for the requested page
 
         Raises:
-            ValueError: If page or page_size is less than 1
+            ValueError: If from_id or count is less than 1
+
+        Cost: with a range-capable storage backend (Basilisk's StableBTreeMap,
+        MemoryStorage) this is one ordered range read per page regardless of
+        how many ids in between were deleted; otherwise one read per id.
         """
-        logger.info(f"Loading entities from {from_id} to {from_id + count}")
+        logger.debug(f"Loading up to {count} entities from id {from_id}")
 
         if from_id < 1:
             raise ValueError("from_id must be at least 1")
         if count < 1:
             raise ValueError("count must be at least 1")
 
-        # Return the slice of entities for the requested page
-        ret = []
+        db = cls.db()
+        type_name = cls.get_full_type_name()
 
-        while len(ret) < count and from_id <= cls.max_id():
-            logger.info(f"Loading entity {from_id}")
-            try:
-                entity = cls.load(str(from_id))
-                if entity:
+        ret: List[T] = []
+        max_id = db.max_id(type_name)
+        while len(ret) < count:
+            need = count - len(ret)
+            page = db.load_page(type_name, from_id, need, max_id=max_id)
+            for entity_id, data in page:
+                try:
+                    # Identity map first, so callers keep getting the same instance
+                    entity = db.get_entity(type_name, entity_id)
+                    if entity is None:
+                        entity = cls._hydrate(entity_id, data)
                     ret.append(entity)
-            except (ValueError, AttributeError) as e:
-                # Skip entities with broken/dangling relation references
-                # (full fix: issue #4 — lazy relation resolution)
-                logger.warning(f"Skipping {cls.__name__}@{from_id}: {e}")
-            from_id += 1
+                except (ValueError, AttributeError) as e:
+                    # Skip entities with broken/dangling relation references
+                    # (full fix: issue #4 — lazy relation resolution)
+                    logger.warning(f"Skipping {cls.__name__}@{entity_id}: {e}")
+            if len(page) < need:
+                break  # no more persisted entities of this type
+            # Page was full but something was skipped: continue after it
+            from_id = int(page[-1][0]) + 1
 
         return ret
 
@@ -674,10 +836,7 @@ class Entity:
                 for attr_name, attr_value in cls.__dict__.items():
                     if attr_name.startswith("_"):
                         continue
-                    if (
-                        isinstance(attr_value, ManyToMany)
-                        and attr_value.unidirectional
-                    ):
+                    if isinstance(attr_value, ManyToMany) and attr_value.unidirectional:
                         my_short = (
                             self._type.split("::")[-1]
                             if "::" in self._type
@@ -692,6 +851,12 @@ class Entity:
                                 self._type, self._id, attr_value.reverse_name
                             )
 
+        # Remove from field indexes (issue #11)
+        for field_name in self._indexed_properties():
+            value = getattr(self, field_name, None)
+            if value is not None:
+                db.field_index_remove(self._type, field_name, str(value), self._id)
+
         db.delete(self._type, self._id)
 
         # Remove from entity registry

ic_python_db/properties.py

@@ -30,12 +30,18 @@ class Property(Generic[T]):
     """Definition of an entity property.
 
     A generic descriptor class that provides type-safe property access.
+
+    With ``indexed=True`` a persistent secondary index (issue #11) maps each
+    distinct value to the list of entity IDs holding it, enabling equality
+    lookups via ``Entity.find_by`` without scanning all entities. ``None``
+    values are not indexed.
     """
 
     name: str
     type: Type[T]
     default: Optional[T]
     validator: Optional[Callable[[T], bool]]
+    indexed: bool
 
     def __init__(
         self,
@@ -43,11 +49,13 @@ class Property(Generic[T]):
         type: Type[T] = type(None),  # type: ignore[assignment]
         default: Optional[T] = None,
         validator: Optional[Callable[[T], bool]] = None,
+        indexed: bool = False,
     ):
         self.name = name
         self.type = type
         self.default = default
         self.validator = validator
+        self.indexed = indexed
 
     def __set_name__(self, owner: type, name: str) -> None:
         self.name = name
@@ -63,6 +71,11 @@ class Property(Generic[T]):
         from .constants import ACTION_CREATE, ACTION_MODIFY
         from .hooks import call_entity_hook
 
+        # Fast path: hydrating from DB — store raw value, indexes are intact.
+        if getattr(obj, "_hydrating", False):
+            obj.__dict__[f"_{PROPERTY_STORAGE_PREFIX}_{self.name}"] = value
+            return
+
         old_value = obj.__dict__.get(
             f"_{PROPERTY_STORAGE_PREFIX}_{self.name}", self.default
         )
@@ -100,6 +113,14 @@ class Property(Generic[T]):
                 raise ValueError(f"Invalid value for {self.name}: {value}")
 
         obj.__dict__[f"_{PROPERTY_STORAGE_PREFIX}_{self.name}"] = value
+
+        if self.indexed and old_value != value and obj._id is not None:
+            db = obj.db()
+            if old_value is not None:
+                db.field_index_remove(obj._type, self.name, str(old_value), obj._id)
+            if value is not None:
+                db.field_index_add(obj._type, self.name, str(value), obj._id)
+
         obj._save()
 
 
@@ -111,6 +132,7 @@ class String(Property[str]):
         min_length: Optional[int] = None,
         max_length: Optional[int] = None,
         default: Optional[str] = None,
+        indexed: bool = False,
     ):
         def validator(value: str) -> bool:
             if min_length is not None and len(value) < min_length:
@@ -119,7 +141,9 @@ class String(Property[str]):
                 return False
             return True
 
-        super().__init__(name="", type=str, default=default, validator=validator)
+        super().__init__(
+            name="", type=str, default=default, validator=validator, indexed=indexed
+        )
 
 
 class Integer(Property[int]):
@@ -130,6 +154,7 @@ class Integer(Property[int]):
         min_value: Optional[int] = None,
         max_value: Optional[int] = None,
         default: Optional[int] = None,
+        indexed: bool = False,
     ):
         def validator(value: int) -> bool:
             if min_value is not None and value < min_value:
@@ -138,7 +163,9 @@ class Integer(Property[int]):
                 return False
             return True
 
-        super().__init__(name="", type=int, default=default, validator=validator)
+        super().__init__(
+            name="", type=int, default=default, validator=validator, indexed=indexed
+        )
 
 
 class Float(Property[float]):
@@ -149,6 +176,7 @@ class Float(Property[float]):
         min_value: Optional[float] = None,
         max_value: Optional[float] = None,
         default: Optional[float] = None,
+        indexed: bool = False,
     ):
         def validator(value: float) -> bool:
             if min_value is not None and value < min_value:
@@ -157,14 +185,16 @@ class Float(Property[float]):
                 return False
             return True
 
-        super().__init__(name="", type=float, default=default, validator=validator)
+        super().__init__(
+            name="", type=float, default=default, validator=validator, indexed=indexed
+        )
 
 
 class Boolean(Property[bool]):
     """Boolean property."""
 
-    def __init__(self, default: Optional[bool] = None):
-        super().__init__(name="", type=bool, default=default)
+    def __init__(self, default: Optional[bool] = None, indexed: bool = False):
+        super().__init__(name="", type=bool, default=default, indexed=indexed)
 
 
 # ── Relation Properties ───────────────────────────────────────────────────────

ic_python_db/storage.py

@@ -3,11 +3,39 @@ Storage backends for IC Python DB
 """
 
 from abc import ABC, abstractmethod
-from typing import Dict, Iterator, Optional, Tuple
+from typing import Dict, Iterator, List, Optional, Tuple
+
+
+def storage_sort_key(key: str) -> Tuple[int, bytes]:
+    """Sort key reproducing the on-chain order of ``str`` keys.
+
+    Basilisk's ``StableBTreeMap`` stores a ``str`` key as
+    ``tag + 4-byte big-endian length + utf-8``, so the B-tree orders keys by
+    byte length first and bytewise second: ``"T@9" < "T@10" < "T@2x"``.
+    ``MemoryStorage.range`` uses this so local tests see exactly the order a
+    canister will.
+    """
+    raw = key.encode("utf-8")
+    return (len(raw), raw)
+
+
+def supports_range(storage) -> bool:
+    """True if ``storage`` offers ``range(start, end, limit)``.
+
+    Basilisk's ``StableBTreeMap`` (>= the version that added ``range``) and
+    ``MemoryStorage`` do; older CDK builds and simple dict-backed test
+    doubles don't, and callers fall back to key probing.
+    """
+    return callable(getattr(storage, "range", None))
 
 
 class Storage(ABC):
-    """Abstract base class for storage backends"""
+    """Abstract base class for storage backends
+
+    ``range`` is optional: backends that can walk keys in order should
+    implement it (see ``storage_sort_key`` for the order that is expected);
+    callers must check ``supports_range`` and fall back to ``get`` probing.
+    """
 
     @abstractmethod
     def insert(self, key: str, value: str) -> None:
@@ -67,3 +95,24 @@ class MemoryStorage(Storage):
     def keys(self) -> Iterator[str]:
         """Return all keys in storage"""
         return iter(self._data.keys())
+
+    def range(
+        self, start: str, end: Optional[str] = None, limit: int = 1000
+    ) -> List[Tuple[str, str]]:
+        """Ordered page of ``(key, value)`` with ``start <= key < end``.
+
+        Same contract as Basilisk's ``StableBTreeMap.range``: half-open,
+        ``end=None`` means unbounded, keys ordered by ``storage_sort_key``.
+        O(n log n) here, which is fine for an in-memory test backend.
+        """
+        if limit < 0:
+            raise ValueError("limit must be >= 0")
+        lo = storage_sort_key(start)
+        hi = None if end is None else storage_sort_key(end)
+        selected = [
+            k
+            for k in self._data
+            if lo <= storage_sort_key(k) and (hi is None or storage_sort_key(k) < hi)
+        ]
+        selected.sort(key=storage_sort_key)
+        return [(k, self._data[k]) for k in selected[:limit]]

{ic_python_db-0.10.0.dist-info → ic_python_db-0.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ic-python-db
-Version: 0.10.0
+Version: 0.12.0
 Summary: A lightweight key-value database written in Python, intended for use on the Internet Computer (IC)
 Home-page: https://github.com/smart-social-contracts/ic-python-db
 Author: Smart Social Contracts
@@ -60,6 +60,7 @@ A lightweight key-value database with entity relationships and audit logging cap
 ## Features
 
 - **Persistent Storage**: Works with StableBTreeMap stable structure for persistent storage on your canister's stable memory so your data persists automatically across canister upgrades.
+- **Range-based paging**: `load_some`, `find`, `instances` and `rebuild_field_index` read a page of entities with a single ordered `range` call on the stable map (Basilisk `StableBTreeMap.range`), so cost tracks the rows returned rather than the highest id ever issued, and deleted ids are free. Backends without `range` (older CDK builds, plain dict test doubles) transparently fall back to one read per id.
 - **Entity-Relational Database**: Create, read and write entities with OneToOne, OneToMany, ManyToOne, and ManyToMany relationships.
 - **Schema Versioning & Upgrade Safety**: Automatic schema introspection, compatibility checking, and auto-migration for safe changes. Breaking changes without a `migrate()` method are rejected, preventing data corruption on canister upgrades.
 - **Entity Hooks**: Intercept and control entity lifecycle events (create, modify, delete) with `on_event` hooks.
@@ -194,6 +195,42 @@ Then use the defined entities to store objects:
 
 For more usage examples, see the [tests](tests/src/tests).
 
+## Indexed Fields
+
+Declare `indexed=True` on a scalar property to maintain a persistent secondary index, enabling equality lookups without scanning all entities:
+
+```python
+class Proposal(Entity):
+    title = String()
+    status = String(max_length=32, indexed=True)
+    org_scope = String(max_length=128, indexed=True)
+
+Proposal(title="A", status="voting", org_scope="Justice")
+Proposal(title="B", status="executed", org_scope="Justice")
+
+# Paginated equality lookup — cost proportional to matches, not max_id
+entities, next_from_id = Proposal.find_by("status", "voting", from_id=1, count=50)
+
+# Count without loading entities
+Proposal.count_by("org_scope", "Justice")  # 2
+```
+
+The index is maintained automatically on create, update, and delete. `None` values are not indexed. Querying a field that is not declared `indexed=True` raises `ValueError`.
+
+### Backfilling existing data
+
+Entities created before a field became indexed are missing from its index. Rebuild in bounded batches (resumable, safe for IC instruction limits — spread calls across multiple update calls or timers):
+
+```python
+cursor = 1
+while cursor is not None:
+    cursor = Proposal.rebuild_field_index("status", from_id=cursor, batch=200)
+```
+
+The rebuild is idempotent. The `indexed` flag is not part of the schema descriptor, so toggling it never triggers upgrade-compatibility errors.
+
+Note: each distinct value's ID list is stored in a single storage entry, so extremely hot values are bounded by the backing store's `max_value_size` — the same constraint as relationship reverse indexes.
+
 ## Namespaces
 
 Organize entities with the `__namespace__` attribute to avoid type conflicts when you have the same class name in different modules:

ic_python_db-0.12.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+ic_python_db/__init__.py,sha256=8F0p76Ez1Hhcb9iHKzJjrCWp4aZGgvAGdP9-b9Ka-CU,990
+ic_python_db/_cdk.py,sha256=zTj8HzXUcPz8R73nh5bo6JMkCorp-KSW1hDutzi5LZs,387
+ic_python_db/constants.py,sha256=oLvKJev-65gXFd284DdYutWBX95sTF5Xkikc6IC4mhM,118
+ic_python_db/context.py,sha256=97F4_EAJm3adOnUfh0SaDisX3nc7F2vFIZwZUsUAy58,1080
+ic_python_db/db_engine.py,sha256=wK6mJshJMjrmVa7LPjA5ZVz1LXSxTtRjht6MU2mDM6k,26133
+ic_python_db/entity.py,sha256=zU_7zoYUs125DToM4Htywc9fhz5NxZhbdvYGMei5RJs,50437
+ic_python_db/hooks.py,sha256=B1wV18dAMH-PootSripmMpG9Cr9M3W3lcNEF7QvPrL4,1620
+ic_python_db/mixins.py,sha256=qZmb3ssXYf2NSjHJtgmdsOlkIJdouDjPHdQ9tZmrZi8,1937
+ic_python_db/properties.py,sha256=l7q3Iv8R-57XMlYwz2gcJV9mObYV_LfR6mw2jZgGwns,27556
+ic_python_db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ic_python_db/schema.py,sha256=h1RAbapExo8pMD5IQvK8ZTVkgyonbPV9qSqliSP0Y14,16249
+ic_python_db/storage.py,sha256=899qyuA3alpAFIPM23A_07vqquUB5ZG5GjjpzlHW1Os,3828
+ic_python_db/system_time.py,sha256=xrx8NfjLXCGsac3hUpuvsNwCH91tQ9jfH4N-XFo8iJk,2614
+ic_python_db-0.12.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
+ic_python_db-0.12.0.dist-info/METADATA,sha256=9AjuP2o844gKy9w2Ee-vSZNrqBrEglF6GvsE3v-JHcI,17115
+ic_python_db-0.12.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+ic_python_db-0.12.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
+ic_python_db-0.12.0.dist-info/RECORD,,

{ic_python_db-0.10.0.dist-info → ic_python_db-0.12.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (83.0.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ic_python_db-0.10.0.dist-info/RECORD

@@ -1,18 +0,0 @@
-ic_python_db/__init__.py,sha256=LH2RcKA378wYJc5qK0lTwMZrNQMUi90Ho2hvcri5gLM,990
-ic_python_db/_cdk.py,sha256=zTj8HzXUcPz8R73nh5bo6JMkCorp-KSW1hDutzi5LZs,387
-ic_python_db/constants.py,sha256=oLvKJev-65gXFd284DdYutWBX95sTF5Xkikc6IC4mhM,118
-ic_python_db/context.py,sha256=97F4_EAJm3adOnUfh0SaDisX3nc7F2vFIZwZUsUAy58,1080
-ic_python_db/db_engine.py,sha256=sA_RkFvgHjaBtvp_lOQ3rr7yuzPJQawBeqwQtn-ZXzQ,18622
-ic_python_db/entity.py,sha256=5ZuPYnR92_vDUfG_jnUPLTz06_bjqtwZQtt9q0Vm5Nc,43648
-ic_python_db/hooks.py,sha256=B1wV18dAMH-PootSripmMpG9Cr9M3W3lcNEF7QvPrL4,1620
-ic_python_db/mixins.py,sha256=qZmb3ssXYf2NSjHJtgmdsOlkIJdouDjPHdQ9tZmrZi8,1937
-ic_python_db/properties.py,sha256=BmW4dWBUn4sGnenlvusp1snER7zvstSDu9drKHM3rns,26413
-ic_python_db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ic_python_db/schema.py,sha256=h1RAbapExo8pMD5IQvK8ZTVkgyonbPV9qSqliSP0Y14,16249
-ic_python_db/storage.py,sha256=2F6m_01g2jwoF40s4BrdO2SffJNSO3gPf0QU3Co7T5Y,1873
-ic_python_db/system_time.py,sha256=xrx8NfjLXCGsac3hUpuvsNwCH91tQ9jfH4N-XFo8iJk,2614
-ic_python_db-0.10.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
-ic_python_db-0.10.0.dist-info/METADATA,sha256=dlFpBN0E5WDXrdK9y4Xz1d-5pIZtcK3x464c-Y5HXpo,15131
-ic_python_db-0.10.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-ic_python_db-0.10.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
-ic_python_db-0.10.0.dist-info/RECORD,,