ic-python-db 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ic_python_db/__init__.py CHANGED
@@ -5,6 +5,7 @@ IC Python DB - A lightweight key-value database with entity relationships and au
5
5
  from .constants import ACTION_CREATE, ACTION_DELETE, ACTION_MODIFY
6
6
  from .db_engine import Database
7
7
  from .entity import Entity
8
+ from .errors import TooLargeError
8
9
  from .mixins import TimestampedMixin
9
10
  from .properties import (
10
11
  Boolean,
@@ -14,13 +15,14 @@ from .properties import (
14
15
  ManyToOne,
15
16
  OneToMany,
16
17
  OneToOne,
18
+ RelationView,
17
19
  String,
18
20
  )
19
21
  from .schema import SchemaIncompatibleError, build_schema, diff_schemas, schema_hash
20
22
  from .storage import MemoryStorage, Storage
21
23
  from .system_time import SystemTime
22
24
 
23
- __version__ = "0.12.0"
25
+ __version__ = "0.13.0"
24
26
  __all__ = [
25
27
  "Database",
26
28
  "Entity",
@@ -40,6 +42,8 @@ __all__ = [
40
42
  "ACTION_MODIFY",
41
43
  "ACTION_DELETE",
42
44
  "SchemaIncompatibleError",
45
+ "TooLargeError",
46
+ "RelationView",
43
47
  "build_schema",
44
48
  "diff_schemas",
45
49
  "schema_hash",
ic_python_db/constants.py CHANGED
@@ -4,3 +4,8 @@ LEVEL_MAX_DEFAULT = 3
4
4
  ACTION_CREATE = "create"
5
5
  ACTION_MODIFY = "modify"
6
6
  ACTION_DELETE = "delete"
7
+
8
+ # Largest collection an unpaged operation (iterating a relation, instances())
9
+ # loads in one call; past it they raise TooLargeError. At about 1M
10
+ # instructions per load this stays well inside a query's 5B budget.
11
+ MAX_UNPAGED = 1000
ic_python_db/db_engine.py CHANGED
@@ -2,18 +2,34 @@
2
2
  Core database engine implementation
3
3
  """
4
4
 
5
+ import hashlib
5
6
  import json
7
+ import sys
6
8
  import time
7
9
  import weakref
8
10
  from typing import Any, Dict, List, Optional, Tuple
9
11
 
10
12
  from ic_python_logging import get_logger
11
13
 
12
- from .storage import MemoryStorage, Storage, supports_range
14
+ from ._cdk import ic
15
+ from .storage import MemoryStorage, Storage, storage_sort_key, supports_range
13
16
 
14
17
  logger = get_logger(__name__)
15
18
 
16
19
  _DIGITS = frozenset("0123456789")
20
+ _HEX = frozenset("0123456789abcdef")
21
+ # Posting-list entry suffix widths (see "Posting lists" below)
22
+ _ID_WIDTH = 12
23
+ _HASH_WIDTH = 24
24
+ _SCAN_PAGE = 500
25
+ _BUCKETS = (("n", _ID_WIDTH, ":"), ("h", _HASH_WIDTH, "g"))
26
+ # Purge queue of deleted index sets (see _ix_clear): fixed-width positions
27
+ # keep its entries contiguous in key order
28
+ _PQ = "_pq@"
29
+ _PQ_HEAD = "_pq:head"
30
+ _PQ_TAIL = "_pq:tail"
31
+ # Entries of deleted sets each index write removes on the side
32
+ _PURGE_PER_WRITE = 20
17
33
 
18
34
 
19
35
  def _is_sequential_id(id_str: str) -> bool:
@@ -45,6 +61,17 @@ def _next_sequential_id(id_str: str, digits: int) -> Optional[int]:
45
61
  return max(int(candidate), 10 ** (digits - 1))
46
62
 
47
63
 
64
+ def _message_time() -> Optional[int]:
65
+ """The IC's time, which is the same throughout one message; None outside
66
+ a canister, where the identity map's weak references do the job."""
67
+ if sys.platform != "wasi" or ic is None:
68
+ return None
69
+ try:
70
+ return ic.time()
71
+ except Exception:
72
+ return None
73
+
74
+
48
75
  class Database:
49
76
  """Main database class providing high-level operations"""
50
77
 
@@ -96,8 +123,15 @@ class Database:
96
123
  )
97
124
 
98
125
  self._entity_types = {}
99
- # Entity registry: {(type_name, entity_id): weakref to entity instance}
100
- self._entity_registry = {}
126
+ # Identity map: {(type_name, entity_id): entity}. Weak, so an entity
127
+ # lives only as long as some caller holds it, and its entry goes with it.
128
+ self._entity_registry = weakref.WeakValueDictionary()
129
+ # In a canister weakref is a stub that holds strong references, so the
130
+ # map is also emptied when a new message starts (see _registry)
131
+ self._registry_time: Optional[int] = None
132
+ # Whether deleted index sets may still have entries to purge (None:
133
+ # not checked yet). Kept in the heap: a query discards it with its writes.
134
+ self._purge_pending = None
101
135
 
102
136
  def as_user(self, user_id: str):
103
137
  """Context manager for running operations as a specific user.
@@ -136,6 +170,7 @@ class Database:
136
170
 
137
171
  # Also clear the entity registry
138
172
  self.clear_registry()
173
+ self._purge_pending = None
139
174
 
140
175
  if not self._db_audit:
141
176
  return
@@ -147,24 +182,28 @@ class Database:
147
182
  self._db_audit.insert("_min_id", "0")
148
183
  self._db_audit.insert("_max_id", "0")
149
184
 
185
+ def _registry(self):
186
+ """The identity map, emptied when a new message starts.
187
+
188
+ ic.time() is constant within one message, so a new value means the
189
+ previous message is over: its entities are dropped from the map
190
+ instead of piling up until the next upgrade. An entity some code
191
+ still holds stays usable, but loading its id gives a new instance.
192
+ """
193
+ now = _message_time()
194
+ if now is not None and now != self._registry_time:
195
+ self._entity_registry.clear()
196
+ self._registry_time = now
197
+ return self._entity_registry
198
+
150
199
  def register_entity(self, entity_instance):
151
200
  """Register an entity instance in the identity map."""
152
201
  key = (entity_instance._type, entity_instance._id)
153
- # Use weak reference to allow garbage collection
154
- self._entity_registry[key] = weakref.ref(entity_instance)
202
+ self._registry()[key] = entity_instance
155
203
 
156
204
  def get_entity(self, type_name: str, entity_id: str):
157
205
  """Get entity from registry if it exists."""
158
- key = (type_name, entity_id)
159
- if key in self._entity_registry:
160
- entity_ref = self._entity_registry[key]
161
- entity = entity_ref() # Get object from weak reference
162
- if entity is not None:
163
- return entity
164
- else:
165
- # Clean up dead reference
166
- del self._entity_registry[key]
167
- return None
206
+ return self._registry().get((type_name, entity_id))
168
207
 
169
208
  def clear_registry(self):
170
209
  """Clear the entity registry (useful for testing)."""
@@ -172,9 +211,7 @@ class Database:
172
211
 
173
212
  def unregister_entity(self, type_name: str, entity_id: str):
174
213
  """Remove an entity from the registry (used when deleting)."""
175
- key = (type_name, entity_id)
176
- if key in self._entity_registry:
177
- del self._entity_registry[key]
214
+ self._registry().pop((type_name, entity_id), None)
178
215
 
179
216
  def _audit(self, op: str, key: str, data: Any) -> None:
180
217
  if self._db_audit and self._audit_enabled:
@@ -493,13 +530,231 @@ class Database:
493
530
 
494
531
  return check_upgrade_compatibility(self, raise_on_error=raise_on_error)
495
532
 
533
+ # ── Posting lists (issue #18) ──────────────────────────────────────────────
534
+ #
535
+ # Field indexes and reverse indexes map one base key to a set of entity
536
+ # ids. Each id is its own storage entry, so adding or removing one costs
537
+ # O(1) storage operations however many ids share the base, and a page is
538
+ # one range read:
539
+ #
540
+ # {base}|n{id zero-padded to _ID_WIDTH} -> id canonical sequential ids
541
+ # {base}|h{sha256(id)[:_HASH_WIDTH]} -> id any other id
542
+ # {base}|c -> number of ids
543
+ #
544
+ # Storage orders str keys by length, then bytewise (storage_sort_key), so
545
+ # fixed-width suffixes keep one base's entries contiguous, and the "n"
546
+ # entries come back in ascending numeric id order.
547
+ #
548
+ # Before 0.13 a base key held the whole set as one JSON array, which hit
549
+ # the value-size limit at ~10k ids. Such an array is converted to entries
550
+ # the first time its base is touched (see _ix_count).
551
+
552
+ def _scan(self, start: str, end: str, limit: int) -> List[Tuple[str, str]]:
553
+ if supports_range(self._db_storage):
554
+ return self._db_storage.range(start, end, limit)
555
+ lo, hi = storage_sort_key(start), storage_sort_key(end)
556
+ keys = sorted(
557
+ (k for k in self._db_storage.keys() if lo <= storage_sort_key(k) < hi),
558
+ key=storage_sort_key,
559
+ )
560
+ return [(k, self._db_storage.get(k)) for k in keys[:limit]]
561
+
562
+ @staticmethod
563
+ def _ix_entry_key(base: str, entity_id: str) -> str:
564
+ if _is_sequential_id(entity_id) and len(entity_id) <= _ID_WIDTH:
565
+ return f"{base}|n{entity_id.zfill(_ID_WIDTH)}"
566
+ digest = hashlib.sha256(entity_id.encode("utf-8")).hexdigest()
567
+ return f"{base}|h{digest[:_HASH_WIDTH]}"
568
+
569
+ def _ix_count(self, base: str) -> int:
570
+ raw = self._db_storage.get(f"{base}|c")
571
+ if raw is not None:
572
+ return int(raw)
573
+ legacy = self._db_storage.get(base)
574
+ if legacy is None:
575
+ return 0
576
+ try:
577
+ ids = json.loads(legacy)
578
+ except ValueError:
579
+ return 0
580
+ if not isinstance(ids, list):
581
+ return 0
582
+ self._db_storage.remove(base)
583
+ unique = {str(i) for i in ids}
584
+ for entity_id in unique:
585
+ self._db_storage.insert(self._ix_entry_key(base, entity_id), entity_id)
586
+ if unique:
587
+ self._db_storage.insert(f"{base}|c", str(len(unique)))
588
+ return len(unique)
589
+
590
+ def _ix_add(self, base: str, entity_id: str) -> None:
591
+ self._purge_some()
592
+ count = self._ix_count(base)
593
+ if count == 0:
594
+ self._ix_revive(base)
595
+ key = self._ix_entry_key(base, entity_id)
596
+ if self._db_storage.get(key) is not None:
597
+ return
598
+ self._db_storage.insert(key, entity_id)
599
+ self._db_storage.insert(f"{base}|c", str(count + 1))
600
+
601
+ def _ix_remove(self, base: str, entity_id: str) -> None:
602
+ self._purge_some()
603
+ count = self._ix_count(base)
604
+ if count == 0:
605
+ return
606
+ key = self._ix_entry_key(base, entity_id)
607
+ if self._db_storage.get(key) is None:
608
+ return
609
+ self._db_storage.remove(key)
610
+ if count > 1:
611
+ self._db_storage.insert(f"{base}|c", str(count - 1))
612
+ else:
613
+ self._db_storage.remove(f"{base}|c")
614
+
615
+ def _ix_contains(self, base: str, entity_id: str) -> bool:
616
+ if self._ix_count(base) == 0:
617
+ return False
618
+ return self._db_storage.get(self._ix_entry_key(base, entity_id)) is not None
619
+
620
+ def _ix_page(self, base: str, from_id: int, count: int) -> List[str]:
621
+ """Up to ``count`` sequential ids >= ``from_id``, ascending."""
622
+ if from_id < 1:
623
+ raise ValueError("from_id must be at least 1")
624
+ if count < 1:
625
+ raise ValueError("count must be at least 1")
626
+ if self._ix_count(base) == 0 or len(str(from_id)) > _ID_WIDTH:
627
+ return []
628
+ start = f"{base}|n{str(from_id).zfill(_ID_WIDTH)}"
629
+ end = f"{base}|n{':' * _ID_WIDTH}"
630
+ cut = len(base) + 2
631
+ # A key of another base can fall inside the range only if its suffix
632
+ # is not all digits (e.g. value "x|n0;..." next to value "x").
633
+ return [
634
+ value
635
+ for key, value in self._scan(start, end, count)
636
+ if all(c in _DIGITS for c in key[cut:])
637
+ ]
638
+
639
+ def _ix_bucket(
640
+ self, base: str, tag: str, width: int, top: str, page_size: int = _SCAN_PAGE
641
+ ):
642
+ start = f"{base}|{tag}{'0' * width}"
643
+ end = f"{base}|{tag}{top * width}"
644
+ cut = len(base) + 2
645
+ valid = _DIGITS if tag == "n" else _HEX
646
+ while True:
647
+ page = self._scan(start, end, page_size)
648
+ for key, value in page:
649
+ if all(c in valid for c in key[cut:]):
650
+ yield key, value
651
+ if len(page) < page_size:
652
+ return
653
+ # Smallest same-length key after the last one: append the lowest
654
+ # char and drop the last, i.e. bump by one in the key order.
655
+ last = page[-1][0]
656
+ start = last[:-1] + chr(ord(last[-1]) + 1)
657
+
658
+ def _ix_all(self, base: str) -> List[str]:
659
+ """Every id in the set: sequential ids ascending, then the others."""
660
+ if self._ix_count(base) == 0:
661
+ return []
662
+ ids = [v for _, v in self._ix_bucket(base, "n", _ID_WIDTH, ":")]
663
+ ids.extend(v for _, v in self._ix_bucket(base, "h", _HASH_WIDTH, "g"))
664
+ return ids
665
+
666
+ def _ix_clear(self, base: str) -> None:
667
+ """Empty a set in O(1); its entries are purged in bounded steps.
668
+
669
+ Every reader checks the counter first, so removing it empties the set
670
+ at once. The entries are queued and removed by later index writes
671
+ (_PURGE_PER_WRITE each) or by purge_step(). ``{base}|d`` marks the set
672
+ as dead and holds its queue position, so re-adding to it (a reused
673
+ custom id) purges the leftovers first instead of reviving them.
674
+ """
675
+ if self._ix_count(base) == 0:
676
+ return
677
+ storage = self._db_storage
678
+ tail = int(storage.get(_PQ_TAIL) or 0)
679
+ storage.insert(self._pq_key(tail), base)
680
+ storage.insert(_PQ_TAIL, str(tail + 1))
681
+ storage.insert(f"{base}|d", str(tail))
682
+ storage.remove(f"{base}|c")
683
+ self._purge_pending = True
684
+
685
+ @staticmethod
686
+ def _pq_key(position: int) -> str:
687
+ return f"{_PQ}{str(position).zfill(_ID_WIDTH)}"
688
+
689
+ def _ix_drain(self, base: str, limit: int) -> int:
690
+ """Remove up to ``limit`` entries of ``base``; returns how many."""
691
+ removed = 0
692
+ for tag, width, top in _BUCKETS:
693
+ keys = []
694
+ page_size = min(limit - removed, _SCAN_PAGE)
695
+ for key, _ in self._ix_bucket(base, tag, width, top, page_size):
696
+ if removed + len(keys) >= limit:
697
+ break
698
+ keys.append(key)
699
+ for key in keys:
700
+ self._db_storage.remove(key)
701
+ removed += len(keys)
702
+ if removed >= limit:
703
+ break
704
+ return removed
705
+
706
+ def _ix_revive(self, base: str) -> None:
707
+ """Before reusing a deleted set: purge its leftovers, drop its queue entry."""
708
+ position = self._db_storage.get(f"{base}|d")
709
+ if position is None:
710
+ return
711
+ while self._ix_drain(base, _SCAN_PAGE) == _SCAN_PAGE:
712
+ pass
713
+ self._db_storage.remove(f"{base}|d")
714
+ queued = self._pq_key(int(position))
715
+ if self._db_storage.get(queued) is not None:
716
+ self._db_storage.remove(queued)
717
+
718
+ def _purge_some(self) -> None:
719
+ if self._purge_pending is not False:
720
+ self.purge_step(_PURGE_PER_WRITE)
721
+
722
+ def purge_step(self, limit: int = 500) -> bool:
723
+ """Remove up to ``limit`` entries of deleted index sets.
724
+
725
+ Deleting an entity empties its relation indexes at once; their entries
726
+ are removed in the background, a few by each later index write. A
727
+ timer can call this to finish sooner. Returns True when nothing is
728
+ left to purge.
729
+ """
730
+ if limit < 1:
731
+ raise ValueError("limit must be at least 1")
732
+ storage = self._db_storage
733
+ head = int(storage.get(_PQ_HEAD) or 0)
734
+ tail = int(storage.get(_PQ_TAIL) or 0)
735
+ start, budget = head, limit
736
+ while head < tail and budget > 0:
737
+ queued = self._pq_key(head)
738
+ base = storage.get(queued)
739
+ removed = self._ix_drain(base, budget) if base is not None else 0
740
+ if removed == budget:
741
+ break # out of budget, maybe not drained yet
742
+ budget -= removed
743
+ if base is not None:
744
+ storage.remove(f"{base}|d")
745
+ storage.remove(queued)
746
+ head += 1
747
+ if head != start:
748
+ storage.insert(_PQ_HEAD, str(head))
749
+ self._purge_pending = head < tail
750
+ return not self._purge_pending
751
+
496
752
  # ── Reverse Index API ──────────────────────────────────────────────────────
497
753
  #
498
754
  # Reverse indexes allow OneToMany / ManyToMany / inverse-OneToOne relations
499
755
  # to be resolved without scanning all child entities.
500
756
  #
501
- # Key format: _ri:{ParentType}:{parent_id}:{relation_name}
502
- # Value: JSON array of child entity IDs, e.g. ["1", "2", "3"]
757
+ # Base key: _ri:{ParentType}:{parent_id}:{relation_name} (posting list)
503
758
 
504
759
  def _ri_key(self, parent_type: str, parent_id: str, relation_name: str) -> str:
505
760
  return f"_ri:{parent_type}:{parent_id}:{relation_name}"
@@ -507,101 +762,98 @@ class Database:
507
762
  def reverse_index_get(
508
763
  self, parent_type: str, parent_id: str, relation_name: str
509
764
  ) -> List[str]:
510
- """Read the reverse index for a parent entity's relation.
765
+ """All child ids of a parent's relation (O(children); prefer
766
+ ``reverse_index_page`` for relations that can grow large)."""
767
+ return self._ix_all(self._ri_key(parent_type, parent_id, relation_name))
511
768
 
512
- Returns:
513
- List of child entity IDs, or empty list if no index exists.
769
+ def reverse_index_page(
770
+ self,
771
+ parent_type: str,
772
+ parent_id: str,
773
+ relation_name: str,
774
+ from_id: int,
775
+ count: int,
776
+ ) -> List[str]:
777
+ """Up to ``count`` sequential child ids >= ``from_id``, ascending.
778
+
779
+ Children with custom (non-sequential) ids are not paged; use
780
+ ``reverse_index_get`` for those.
514
781
  """
515
- key = self._ri_key(parent_type, parent_id, relation_name)
516
- raw = self._db_storage.get(key)
517
- if raw:
518
- return json.loads(raw)
519
- return []
782
+ return self._ix_page(
783
+ self._ri_key(parent_type, parent_id, relation_name), from_id, count
784
+ )
785
+
786
+ def reverse_index_count(
787
+ self, parent_type: str, parent_id: str, relation_name: str
788
+ ) -> int:
789
+ """Number of child ids in a parent's relation (one read)."""
790
+ return self._ix_count(self._ri_key(parent_type, parent_id, relation_name))
791
+
792
+ def reverse_index_contains(
793
+ self, parent_type: str, parent_id: str, relation_name: str, child_id: str
794
+ ) -> bool:
795
+ return self._ix_contains(
796
+ self._ri_key(parent_type, parent_id, relation_name), child_id
797
+ )
520
798
 
521
799
  def reverse_index_add(
522
800
  self, parent_type: str, parent_id: str, relation_name: str, child_id: str
523
801
  ) -> None:
524
802
  """Add a child ID to a parent's reverse index."""
525
- key = self._ri_key(parent_type, parent_id, relation_name)
526
- raw = self._db_storage.get(key)
527
- ids = json.loads(raw) if raw else []
528
- if child_id not in ids:
529
- ids.append(child_id)
530
- self._db_storage.insert(key, json.dumps(ids))
803
+ self._ix_add(self._ri_key(parent_type, parent_id, relation_name), child_id)
531
804
 
532
805
  def reverse_index_remove(
533
806
  self, parent_type: str, parent_id: str, relation_name: str, child_id: str
534
807
  ) -> None:
535
808
  """Remove a child ID from a parent's reverse index."""
536
- key = self._ri_key(parent_type, parent_id, relation_name)
537
- raw = self._db_storage.get(key)
538
- if not raw:
539
- return
540
- ids = json.loads(raw)
541
- if child_id in ids:
542
- ids.remove(child_id)
543
- if ids:
544
- self._db_storage.insert(key, json.dumps(ids))
545
- else:
546
- self._db_storage.remove(key)
809
+ self._ix_remove(self._ri_key(parent_type, parent_id, relation_name), child_id)
547
810
 
548
811
  def reverse_index_delete(
549
812
  self, parent_type: str, parent_id: str, relation_name: str
550
813
  ) -> None:
551
- """Delete an entire reverse index entry (used when parent is deleted)."""
552
- key = self._ri_key(parent_type, parent_id, relation_name)
553
- if self._db_storage.get(key) is not None:
554
- self._db_storage.remove(key)
814
+ """Delete a whole reverse index (used when the parent is deleted).
815
+
816
+ Costs one removal per child id.
817
+ """
818
+ self._ix_clear(self._ri_key(parent_type, parent_id, relation_name))
555
819
 
556
820
  # ── Field Index API (issue #11) ────────────────────────────────────────────
557
821
  #
558
822
  # Secondary indexes for scalar properties declared with indexed=True.
559
823
  # Allow equality lookups (Entity.find_by) without scanning all entities.
560
824
  #
561
- # Key format: _fi:{Type}:{field}:{value}
562
- # Value: JSON array of entity IDs, e.g. ["1", "2", "3"]
825
+ # Base key: _fi:{Type}:{field}:{value} (posting list)
563
826
 
564
827
  def _fi_key(self, type_name: str, field_name: str, value: str) -> str:
565
828
  return f"_fi:{type_name}:{field_name}:{value}"
566
829
 
567
830
  def field_index_get(self, type_name: str, field_name: str, value: str) -> List[str]:
568
- """Read the ID list for entities whose ``field_name`` equals ``value``.
831
+ """All ids of entities whose ``field_name`` equals ``value``
832
+ (O(matches); prefer ``field_index_page``)."""
833
+ return self._ix_all(self._fi_key(type_name, field_name, value))
569
834
 
570
- Returns:
571
- List of entity IDs, or empty list if no index entry exists.
572
- """
573
- key = self._fi_key(type_name, field_name, value)
574
- raw = self._db_storage.get(key)
575
- if raw:
576
- return json.loads(raw)
577
- return []
835
+ def field_index_page(
836
+ self, type_name: str, field_name: str, value: str, from_id: int, count: int
837
+ ) -> List[str]:
838
+ """Up to ``count`` sequential ids >= ``from_id`` whose ``field_name``
839
+ equals ``value``, ascending. One range read."""
840
+ return self._ix_page(self._fi_key(type_name, field_name, value), from_id, count)
841
+
842
+ def field_index_count(self, type_name: str, field_name: str, value: str) -> int:
843
+ """Number of entities whose ``field_name`` equals ``value`` (one read)."""
844
+ return self._ix_count(self._fi_key(type_name, field_name, value))
578
845
 
579
846
  def field_index_add(
580
847
  self, type_name: str, field_name: str, value: str, entity_id: str
581
848
  ) -> None:
582
849
  """Add an entity ID to the index entry for ``value``."""
583
- key = self._fi_key(type_name, field_name, value)
584
- raw = self._db_storage.get(key)
585
- ids = json.loads(raw) if raw else []
586
- if entity_id not in ids:
587
- ids.append(entity_id)
588
- self._db_storage.insert(key, json.dumps(ids))
850
+ self._ix_add(self._fi_key(type_name, field_name, value), entity_id)
589
851
 
590
852
  def field_index_remove(
591
853
  self, type_name: str, field_name: str, value: str, entity_id: str
592
854
  ) -> None:
593
855
  """Remove an entity ID from the index entry for ``value``."""
594
- key = self._fi_key(type_name, field_name, value)
595
- raw = self._db_storage.get(key)
596
- if not raw:
597
- return
598
- ids = json.loads(raw)
599
- if entity_id in ids:
600
- ids.remove(entity_id)
601
- if ids:
602
- self._db_storage.insert(key, json.dumps(ids))
603
- else:
604
- self._db_storage.remove(key)
856
+ self._ix_remove(self._fi_key(type_name, field_name, value), entity_id)
605
857
 
606
858
  # Reverse counters (unidirectional relations)
607
859
  # Key format: "_rc:{parent_type}:{parent_id}:{relation_name}"
@@ -620,41 +872,32 @@ class Database:
620
872
  ) -> int:
621
873
  """Read the reverse counter for a parent entity's relation.
622
874
 
623
- Falls back to the length of a legacy reverse-index array (from when
624
- the relation was bidirectional) if no counter exists yet.
875
+ Falls back to the size of a legacy reverse index (from when the
876
+ relation was bidirectional) if no counter exists yet.
625
877
  """
626
878
  key = self._rc_key(parent_type, parent_id, relation_name)
627
879
  raw = self._db_storage.get(key)
628
880
  if raw is not None:
629
881
  return int(raw)
630
- legacy = self._db_storage.get(
631
- self._ri_key(parent_type, parent_id, relation_name)
632
- )
633
- if legacy:
634
- return len(json.loads(legacy))
635
- return 0
882
+ return self.reverse_index_count(parent_type, parent_id, relation_name)
636
883
 
637
884
  def reverse_count_add(
638
885
  self, parent_type: str, parent_id: str, relation_name: str, delta: int
639
886
  ) -> None:
640
887
  """Adjust the reverse counter by delta (never below zero).
641
888
 
642
- On first write, seeds the counter from a legacy reverse-index array
643
- (if present) and drops the array — lazy, exactly-once migration of
644
- relations converted from bidirectional to unidirectional.
889
+ On first write, seeds the counter from a legacy reverse index (if
890
+ present) and drops it — lazy, exactly-once migration of relations
891
+ converted from bidirectional to unidirectional.
645
892
  """
646
893
  key = self._rc_key(parent_type, parent_id, relation_name)
647
894
  raw = self._db_storage.get(key)
648
895
  if raw is not None:
649
896
  current = int(raw)
650
897
  else:
651
- legacy_key = self._ri_key(parent_type, parent_id, relation_name)
652
- legacy = self._db_storage.get(legacy_key)
653
- if legacy:
654
- current = len(json.loads(legacy))
655
- self._db_storage.remove(legacy_key)
656
- else:
657
- current = 0
898
+ current = self.reverse_index_count(parent_type, parent_id, relation_name)
899
+ if current:
900
+ self.reverse_index_delete(parent_type, parent_id, relation_name)
658
901
  self._db_storage.insert(key, str(max(0, current + delta)))
659
902
 
660
903
  def reverse_count_delete(
ic_python_db/entity.py CHANGED
@@ -80,7 +80,7 @@ class Entity:
80
80
  Entities with namespaces are stored as "namespace::ClassName"
81
81
  Allows multiple entities with same class name in different namespaces
82
82
  _entity_type (str): Optional entity type for subclasses
83
- _context (Set[Entity]): Set of all entities in current context
83
+ _context (Set[Entity]): Unused, kept empty for compatibility (issue #23)
84
84
 
85
85
  Property Storage:
86
86
  User-defined properties (String, Integer, etc.) are stored as _prop_{name}
@@ -89,7 +89,10 @@ class Entity:
89
89
  """
90
90
 
91
91
  _entity_type = None # To be defined in subclasses
92
- _context: Set["Entity"] = set() # Set of entities in current context
92
+ # Used to hold every entity ever constructed, which kept them all in the
93
+ # heap until the next upgrade (issue #23). Kept, empty, for code that
94
+ # clears it.
95
+ _context: Set["Entity"] = set()
93
96
  _deferred_types: List[Type["Entity"]] = [] # Types defined before DB exists
94
97
  _do_not_save = False
95
98
  __version__ = 1 # Default schema version
@@ -160,9 +163,6 @@ class Entity:
160
163
  self._loaded = False if kwargs.get("_loaded") is None else kwargs["_loaded"]
161
164
  self._counted = False # Track if this entity has been counted
162
165
 
163
- # Add to context
164
- self.__class__._context.add(self)
165
-
166
166
  # Register this type with the database (using full type name)
167
167
  self.db().register_entity_type(self.__class__, self._type)
168
168
 
@@ -457,12 +457,42 @@ class Entity:
457
457
 
458
458
  return entity
459
459
 
460
+ @classmethod
461
+ def _require_unpaged(cls, method: str) -> None:
462
+ """Raise TooLargeError before ``method`` loads more than MAX_UNPAGED entities."""
463
+ from . import constants
464
+ from .errors import TooLargeError
465
+
466
+ total = cls.count()
467
+ if total > constants.MAX_UNPAGED:
468
+ raise TooLargeError(
469
+ f"{cls.__name__}.{method}() would load all {total} entities, more "
470
+ f"than {constants.MAX_UNPAGED} in one call; use {cls.__name__}.count(), "
471
+ f"{cls.__name__}.find_by() on an indexed field, or "
472
+ f"{cls.__name__}.page(from_id, count) instead"
473
+ )
474
+
460
475
  @classmethod
461
476
  def find(cls: Type[T], d) -> List[T]:
462
477
  """Find entities matching a dictionary filter.
463
478
 
464
- Uses load_some over the full ID range. Prefer indexed lookups where possible.
479
+ .. deprecated::
480
+ Loads every entity of the type to filter them, and raises
481
+ TooLargeError past constants.MAX_UNPAGED. Use find_by() on an
482
+ indexed field, or page() (issue #22).
465
483
  """
484
+ try:
485
+ import warnings
486
+
487
+ warnings.warn(
488
+ "Entity.find() is deprecated and will be removed. "
489
+ "Use find_by() on an indexed field, or page().",
490
+ DeprecationWarning,
491
+ stacklevel=2,
492
+ )
493
+ except (ImportError, AttributeError):
494
+ pass
495
+ cls._require_unpaged("find")
466
496
  max_id = cls.max_id()
467
497
  if max_id == 0:
468
498
  return []
@@ -505,9 +535,11 @@ class Entity:
505
535
  ) -> Tuple[List[T], Optional[int]]:
506
536
  """Equality lookup on an indexed field with cursor pagination (issue #11).
507
537
 
508
- IDs come from the persistent field index, so cost is proportional to
509
- the number of matches, not to max_id. Results are ordered by ascending
510
- entity ID, compatible with load_some-style cursors.
538
+ IDs come from the persistent field index with one range read per
539
+ page, so cost is proportional to ``count``, not to the number of
540
+ matches or to max_id. Results are ordered by ascending entity ID,
541
+ compatible with load_some-style cursors. Entities with custom
542
+ (non-sequential) ids are not returned.
511
543
 
512
544
  Args:
513
545
  field: Name of a property declared with indexed=True
@@ -528,34 +560,32 @@ class Entity:
528
560
  if count < 1:
529
561
  raise ValueError("count must be at least 1")
530
562
 
531
- ids = cls.db().field_index_get(cls.get_full_type_name(), field, str(value))
532
- matching = sorted((int(i) for i in ids if i.isdigit() and int(i) >= from_id))
533
-
563
+ db = cls.db()
564
+ type_name = cls.get_full_type_name()
534
565
  entities: List[T] = []
535
- last_id = None
536
- for entity_id in matching:
537
- if len(entities) >= count:
538
- break
539
- entity = cls.load(str(entity_id))
540
- if entity is not None:
541
- entities.append(entity)
542
- last_id = entity_id
543
-
544
- has_more = last_id is not None and any(i > last_id for i in matching)
545
- next_from_id = (last_id + 1) if has_more else None
546
- return entities, next_from_id
566
+ cursor = from_id
567
+ while len(entities) < count:
568
+ need = count - len(entities)
569
+ # One extra id tells whether another page exists
570
+ ids = db.field_index_page(type_name, field, str(value), cursor, need + 1)
571
+ for entity_id in ids[:need]:
572
+ entity = cls.load(entity_id)
573
+ if entity is not None:
574
+ entities.append(entity)
575
+ if len(ids) <= need:
576
+ return entities, None
577
+ cursor = int(ids[need])
578
+ return entities, cursor
547
579
 
548
580
  @classmethod
549
581
  def count_by(cls: Type[T], field: str, value: Any) -> int:
550
- """Number of entities whose indexed ``field`` equals ``value``.
582
+ """Number of entities whose indexed ``field`` equals ``value`` (one read).
551
583
 
552
584
  Raises:
553
585
  ValueError: If the field is not declared with indexed=True
554
586
  """
555
587
  cls._require_indexed_field(field)
556
- return len(
557
- cls.db().field_index_get(cls.get_full_type_name(), field, str(value))
558
- )
588
+ return cls.db().field_index_count(cls.get_full_type_name(), field, str(value))
559
589
 
560
590
  @classmethod
561
591
  def rebuild_field_index(
@@ -612,8 +642,9 @@ class Entity:
612
642
  """Get all instances of this entity type, including subclass instances.
613
643
 
614
644
  .. deprecated::
615
- This method loads ALL entities and is O(max_id). Prefer load_some()
616
- with pagination or use relationship accessors.
645
+ Loads every entity, and raises TooLargeError past
646
+ constants.MAX_UNPAGED. Use count(), find_by() on an indexed field,
647
+ or page() (issue #22).
617
648
 
618
649
  Returns:
619
650
  List of entities
@@ -623,7 +654,7 @@ class Entity:
623
654
 
624
655
  warnings.warn(
625
656
  "Entity.instances() is deprecated and will be removed. "
626
- "Use load_some() with pagination instead.",
657
+ "Use count(), find_by() or page() instead.",
627
658
  DeprecationWarning,
628
659
  stacklevel=2,
629
660
  )
@@ -632,6 +663,7 @@ class Entity:
632
663
  db = Database.get_instance()
633
664
  full_type_name = cls.get_full_type_name()
634
665
  db.register_entity_type(cls, full_type_name)
666
+ cls._require_unpaged("instances")
635
667
 
636
668
  max_id = cls.max_id()
637
669
  if max_id == 0:
@@ -653,6 +685,44 @@ class Entity:
653
685
 
654
686
  return instances
655
687
 
688
+ @classmethod
689
+ def page(
690
+ cls: Type[T], from_id: int = 1, count: int = 50
691
+ ) -> Tuple[List[T], Optional[int]]:
692
+ """One page of this type's entities, in ascending id order (issue #22).
693
+
694
+ Returns (entities, next_from_id); next_from_id is None on the last page.
695
+ Costs one range read plus ``count`` loads, however many entities
696
+ exist, so it replaces instances() for large types::
697
+
698
+ cursor = 1
699
+ while cursor is not None:
700
+ users, cursor = User.page(cursor, 50)
701
+
702
+ Only this type's own entities (not subclasses'), with canonical
703
+ sequential ids, as with load_some(). Entities that fail to load are
704
+ skipped, so a page may hold fewer than ``count``.
705
+ """
706
+ if from_id < 1:
707
+ raise ValueError("from_id must be at least 1")
708
+ if count < 1:
709
+ raise ValueError("count must be at least 1")
710
+ db = cls.db()
711
+ type_name = cls.get_full_type_name()
712
+ # One extra row tells whether another page exists
713
+ rows = db.load_page(type_name, from_id, count + 1)
714
+ entities: List[T] = []
715
+ for entity_id, data in rows[:count]:
716
+ try:
717
+ entity = db.get_entity(type_name, entity_id)
718
+ if entity is None:
719
+ entity = cls._hydrate(entity_id, data)
720
+ entities.append(entity)
721
+ except (ValueError, AttributeError) as e:
722
+ logger.warning(f"Skipping {cls.__name__}@{entity_id}: {e}")
723
+ next_from_id = int(rows[count][0]) if len(rows) > count else None
724
+ return entities, next_from_id
725
+
656
726
  @classmethod
657
727
  def count(cls: Type[T]) -> int:
658
728
  """Get the total count of entities of this type.
@@ -749,7 +819,13 @@ class Entity:
749
819
  logger.debug(f"Deleting entity {self._type}@{self._id}")
750
820
  from .constants import ACTION_DELETE
751
821
  from .hooks import call_entity_hook
752
- from .properties import ManyToMany, ManyToOne, OneToMany, OneToOne
822
+ from .properties import (
823
+ ManyToMany,
824
+ ManyToOne,
825
+ OneToMany,
826
+ OneToOne,
827
+ _require_unpaged,
828
+ )
753
829
 
754
830
  # Call hook before deletion
755
831
  allow, _ = call_entity_hook(self, None, self, None, ACTION_DELETE)
@@ -759,6 +835,18 @@ class Entity:
759
835
 
760
836
  db = self.db()
761
837
 
838
+ # Refuse before changing anything: each ManyToMany member is updated
839
+ for cls in reversed(self.__class__.__mro__):
840
+ for attr_name, attr_value in cls.__dict__.items():
841
+ if not attr_name.startswith("_") and isinstance(attr_value, ManyToMany):
842
+ _require_unpaged(
843
+ db,
844
+ self,
845
+ attr_name,
846
+ "delete it",
847
+ f"first .discard() its {attr_name} page by page",
848
+ )
849
+
762
850
  # Clean up reverse indexes for all relation properties
763
851
  for cls in reversed(self.__class__.__mro__):
764
852
  for attr_name, attr_value in cls.__dict__.items():
@@ -883,9 +971,6 @@ class Entity:
883
971
 
884
972
  logger.debug(f"Deleted entity {self._type}@{self._id}")
885
973
 
886
- # Remove from context
887
- self.__class__._context.discard(self)
888
-
889
974
  def _serialize_base(self) -> Dict[str, Any]:
890
975
  """Shared serialization logic: core data, properties, and instance attributes."""
891
976
  # Get mixin data first if available
@@ -963,7 +1048,12 @@ class Entity:
963
1048
  """
964
1049
  data = self._serialize_base()
965
1050
 
966
- from ic_python_db.properties import ManyToMany, ManyToOne, OneToOne
1051
+ from ic_python_db.properties import (
1052
+ ManyToMany,
1053
+ ManyToOne,
1054
+ OneToOne,
1055
+ _require_unpaged,
1056
+ )
967
1057
 
968
1058
  db = self.db()
969
1059
 
@@ -984,6 +1074,13 @@ class Entity:
984
1074
  ref, attr_value._get_allowed_types()
985
1075
  )
986
1076
  elif isinstance(attr_value, ManyToMany):
1077
+ _require_unpaged(
1078
+ db,
1079
+ self,
1080
+ attr_name,
1081
+ "serialize it",
1082
+ f"serialize its {attr_name} with .page()",
1083
+ )
987
1084
  related_ids = db.reverse_index_get(self._type, self._id, attr_name)
988
1085
  if related_ids:
989
1086
  data[attr_name] = [
ic_python_db/errors.py ADDED
@@ -0,0 +1,10 @@
1
+ """Errors raised by ic_python_db."""
2
+
3
+
4
+ class TooLargeError(ValueError):
5
+ """An operation would touch more than ``constants.MAX_UNPAGED`` items in one call.
6
+
7
+ On the IC each call has a fixed instruction budget, and running past it
8
+ traps with no hint of the cause. This error is raised first instead, and
9
+ its message names the paged alternative.
10
+ """
@@ -373,8 +373,8 @@ class ManyToOne(Relation[E]):
373
373
  class OneToMany(Relation[E]):
374
374
  """One-to-many relationship (parent side, resolved via reverse index).
375
375
 
376
- Accessing this property reads the persisted reverse index to find child
377
- IDs, then loads each child. No scanning required.
376
+ Accessing this property returns a RelationView: counting, membership and
377
+ pages read the persisted reverse index without loading every child.
378
378
 
379
379
  Example:
380
380
  class Department(Entity):
@@ -389,25 +389,10 @@ class OneToMany(Relation[E]):
389
389
 
390
390
  def __get__(
391
391
  self, obj: object, objtype: Optional[type] = None
392
- ) -> Union["OneToMany[E]", List[E]]:
392
+ ) -> Union["OneToMany[E]", "RelationView"]:
393
393
  if obj is None:
394
394
  return self # type: ignore[return-value]
395
-
396
- from .db_engine import Database
397
-
398
- db = Database.get_instance()
399
- child_ids = db.reverse_index_get(obj._type, obj._id, self.name)
400
-
401
- children = []
402
- for child_id in child_ids:
403
- for type_name in self._get_allowed_types():
404
- entity_class = db._entity_types.get(type_name)
405
- if entity_class:
406
- child = entity_class.load(child_id)
407
- if child:
408
- children.append(child)
409
- break
410
- return children # type: ignore[return-value]
395
+ return RelationView(obj, self) # type: ignore[return-value]
411
396
 
412
397
  def __set__(self, obj, values):
413
398
  raise AttributeError(
@@ -556,19 +541,154 @@ class OneToOne(Relation[E]):
556
541
  obj._save()
557
542
 
558
543
 
559
- class ManyToManyList(list):
560
- """List subclass returned by ManyToMany.__get__ that supports .add() and .remove()."""
544
+ def _require_unpaged(db, obj, relation_name, action, instead):
545
+ """Raise TooLargeError before ``action`` loads more members than MAX_UNPAGED."""
546
+ from . import constants
547
+ from .errors import TooLargeError
548
+
549
+ total = db.reverse_index_count(obj._type, obj._id, relation_name)
550
+ if total > constants.MAX_UNPAGED:
551
+ owner_type = obj._type.split("::")[-1]
552
+ raise TooLargeError(
553
+ f"{owner_type}.{relation_name} has {total} members, more than "
554
+ f"{constants.MAX_UNPAGED} can be handled in one call; to {action}, {instead}"
555
+ )
556
+
557
+
558
+ class RelationView:
559
+ """The members of a OneToMany or ManyToMany relation, loaded only on demand.
560
+
561
+ Returned by the relation's getter instead of a list (issue #21), so that
562
+ questions about a large relation stay cheap:
563
+
564
+ if section.stands: ... # one read
565
+ len(section.stands) # one read
566
+ stand in section.stands # two reads
567
+ page, cursor = section.stands.page(1, 50) # one range read + 50 loads
568
+ group.members.add(user) # index writes only, no loads
561
569
 
562
- def __init__(self, items, owner, prop):
563
- super().__init__(items)
570
+ Iterating, indexing and comparing load every member and raise
571
+ TooLargeError past constants.MAX_UNPAGED; use page() for large relations.
572
+ ``list(view)`` gives a plain list.
573
+
574
+ len() and ``in`` count stored references. A unidirectional target deleted
575
+ elsewhere stays referenced until discarded (no reverse list exists to find
576
+ its owners); loading skips it.
577
+ """
578
+
579
+ def __init__(self, owner, prop):
564
580
  self._owner = owner
565
581
  self._prop = prop
566
582
 
583
+ def _db(self):
584
+ from .db_engine import Database
585
+
586
+ return Database.get_instance()
587
+
588
+ def _key(self):
589
+ return (self._owner._type, self._owner._id, self._prop.name)
590
+
591
+ def _load(self, entity_id):
592
+ db = self._db()
593
+ for type_name in self._prop._get_allowed_types():
594
+ entity_class = db._entity_types.get(type_name)
595
+ if entity_class:
596
+ entity = entity_class.load(entity_id)
597
+ if entity:
598
+ return entity
599
+ return None
600
+
601
+ def __len__(self):
602
+ return self._db().reverse_index_count(*self._key())
603
+
604
+ def __bool__(self):
605
+ return len(self) > 0
606
+
607
+ def __contains__(self, entity):
608
+ from .entity import Entity
609
+
610
+ if isinstance(entity, Entity):
611
+ try:
612
+ self._prop.validate_entity(entity)
613
+ except TypeError:
614
+ return False
615
+ entity_id = entity._id
616
+ else:
617
+ entity_id = str(entity)
618
+ return self._db().reverse_index_contains(*self._key(), entity_id)
619
+
620
+ def page(self, from_id: int = 1, count: int = 50):
621
+ """Up to ``count`` members with id >= ``from_id``, in ascending id order.
622
+
623
+ Returns (members, next_from_id); next_from_id is None on the last page.
624
+ Costs one range read and ``count`` loads, however large the relation.
625
+ Members with custom (non-sequential) ids are not paged.
626
+ """
627
+ if count < 1:
628
+ raise ValueError("count must be at least 1")
629
+ db = self._db()
630
+ members = []
631
+ cursor = from_id
632
+ while len(members) < count:
633
+ need = count - len(members)
634
+ # One extra id tells whether another page exists
635
+ ids = db.reverse_index_page(*self._key(), cursor, need + 1)
636
+ for entity_id in ids[:need]:
637
+ entity = self._load(entity_id)
638
+ if entity is not None:
639
+ members.append(entity)
640
+ if len(ids) <= need:
641
+ return members, None
642
+ cursor = int(ids[need])
643
+ return members, cursor
644
+
645
+ def _all(self):
646
+ from . import constants
647
+ from .errors import TooLargeError
648
+
649
+ total = len(self)
650
+ if total > constants.MAX_UNPAGED:
651
+ owner_type = self._owner._type.split("::")[-1]
652
+ raise TooLargeError(
653
+ f"{owner_type}.{self._prop.name} has {total} members, more than "
654
+ f"{constants.MAX_UNPAGED} can be loaded in one call; use "
655
+ f".page(from_id, count), len() or 'in' instead"
656
+ )
657
+ members = []
658
+ for entity_id in self._db().reverse_index_get(*self._key()):
659
+ entity = self._load(entity_id)
660
+ if entity is not None:
661
+ members.append(entity)
662
+ return members
663
+
664
+ def __iter__(self):
665
+ return iter(self._all())
666
+
667
+ def __getitem__(self, index):
668
+ return self._all()[index]
669
+
670
+ def __eq__(self, other):
671
+ if isinstance(other, RelationView):
672
+ other = other._all()
673
+ if isinstance(other, (list, tuple)):
674
+ return self._all() == list(other)
675
+ return NotImplemented
676
+
677
+ def __repr__(self):
678
+ return f"<RelationView {self._owner._type}@{self._owner._id}.{self._prop.name}>"
679
+
680
+ def _require_many_to_many(self, action):
681
+ if not isinstance(self._prop, ManyToMany):
682
+ raise TypeError(
683
+ f"Cannot {action} on OneToMany '{self._prop.name}'. "
684
+ f"Set the ManyToOne '{self._prop.reverse_name}' on the child instead."
685
+ )
686
+
567
687
  def add(self, entity):
568
688
  """Add an entity to this many-to-many relationship."""
569
- from .db_engine import Database
570
689
  from .entity import Entity
571
690
 
691
+ self._require_many_to_many("add")
572
692
  if isinstance(entity, (str, int)):
573
693
  resolved = self._prop.resolve_entity(self._owner, entity)
574
694
  elif isinstance(entity, Entity):
@@ -577,15 +697,10 @@ class ManyToManyList(list):
577
697
  raise TypeError(f"Cannot add {type(entity)} to ManyToMany relation")
578
698
 
579
699
  self._prop.validate_entity(resolved)
580
- db = Database.get_instance()
700
+ db = self._db()
581
701
 
582
- existing = db.reverse_index_get(
583
- self._owner._type, self._owner._id, self._prop.name
584
- )
585
- if resolved._id not in existing:
586
- db.reverse_index_add(
587
- self._owner._type, self._owner._id, self._prop.name, resolved._id
588
- )
702
+ if not db.reverse_index_contains(*self._key(), resolved._id):
703
+ db.reverse_index_add(*self._key(), resolved._id)
589
704
  if self._prop.unidirectional:
590
705
  db.reverse_count_add(
591
706
  resolved._type, resolved._id, self._prop.reverse_name, 1
@@ -599,13 +714,12 @@ class ManyToManyList(list):
599
714
  )
600
715
  if not self._owner._do_not_save:
601
716
  self._owner._save()
602
- self.append(resolved)
603
717
 
604
718
  def discard(self, entity):
605
719
  """Remove an entity from this many-to-many relationship (no error if absent)."""
606
- from .db_engine import Database
607
720
  from .entity import Entity
608
721
 
722
+ self._require_many_to_many("discard")
609
723
  if isinstance(entity, Entity):
610
724
  entity_id = entity._id
611
725
  entity_type = entity._type
@@ -613,13 +727,9 @@ class ManyToManyList(list):
613
727
  entity_id = str(entity)
614
728
  entity_type = None
615
729
 
616
- db = Database.get_instance()
617
- was_present = entity_id in db.reverse_index_get(
618
- self._owner._type, self._owner._id, self._prop.name
619
- )
620
- db.reverse_index_remove(
621
- self._owner._type, self._owner._id, self._prop.name, entity_id
622
- )
730
+ db = self._db()
731
+ was_present = db.reverse_index_contains(*self._key(), entity_id)
732
+ db.reverse_index_remove(*self._key(), entity_id)
623
733
  if self._prop.unidirectional:
624
734
  if was_present:
625
735
  if entity_type is None:
@@ -641,7 +751,6 @@ class ManyToManyList(list):
641
751
  type_name, entity_id, self._prop.reverse_name, self._owner._id
642
752
  )
643
753
 
644
- self[:] = [e for e in self if getattr(e, "_id", None) != entity_id]
645
754
  if not self._owner._do_not_save:
646
755
  self._owner._save()
647
756
 
@@ -650,6 +759,10 @@ class ManyToManyList(list):
650
759
  self.discard(entity)
651
760
 
652
761
 
762
+ # Name of the view before 0.13, when it was a list subclass
763
+ ManyToManyList = RelationView
764
+
765
+
653
766
  class ManyToMany(Relation[E]):
654
767
  """Many-to-many relationship with bidirectional reverse indexes.
655
768
 
@@ -690,25 +803,10 @@ class ManyToMany(Relation[E]):
690
803
 
691
804
  def __get__(
692
805
  self, obj: object, objtype: Optional[type] = None
693
- ) -> Union["ManyToMany[E]", List[E]]:
806
+ ) -> Union["ManyToMany[E]", "RelationView"]:
694
807
  if obj is None:
695
808
  return self # type: ignore[return-value]
696
-
697
- from .db_engine import Database
698
-
699
- db = Database.get_instance()
700
- related_ids = db.reverse_index_get(obj._type, obj._id, self.name)
701
-
702
- entities = []
703
- for related_id in related_ids:
704
- for type_name in self._get_allowed_types():
705
- entity_class = db._entity_types.get(type_name)
706
- if entity_class:
707
- entity = entity_class.load(related_id)
708
- if entity:
709
- entities.append(entity)
710
- break
711
- return ManyToManyList(entities, obj, self) # type: ignore[return-value]
809
+ return RelationView(obj, self) # type: ignore[return-value]
712
810
 
713
811
  def __set__(self, obj, values):
714
812
  from .db_engine import Database
@@ -730,6 +828,7 @@ class ManyToMany(Relation[E]):
730
828
  resolved.append(entity)
731
829
 
732
830
  db = Database.get_instance()
831
+ _require_unpaged(db, obj, self.name, "set it", "use .add() and .discard()")
733
832
 
734
833
  # Remove old relations from both sides' indexes
735
834
  old_ids = db.reverse_index_get(obj._type, obj._id, self.name)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ic-python-db
3
- Version: 0.12.0
3
+ Version: 0.13.0
4
4
  Summary: A lightweight key-value database written in Python, intended for use on the Internet Computer (IC)
5
5
  Home-page: https://github.com/smart-social-contracts/ic-python-db
6
6
  Author: Smart Social Contracts
@@ -60,7 +60,7 @@ A lightweight key-value database with entity relationships and audit logging cap
60
60
  ## Features
61
61
 
62
62
  - **Persistent Storage**: Works with StableBTreeMap stable structure for persistent storage on your canister's stable memory so your data persists automatically across canister upgrades.
63
- - **Range-based paging**: `load_some`, `find`, `instances` and `rebuild_field_index` read a page of entities with a single ordered `range` call on the stable map (Basilisk `StableBTreeMap.range`), so cost tracks the rows returned rather than the highest id ever issued, and deleted ids are free. Backends without `range` (older CDK builds, plain dict test doubles) transparently fall back to one read per id.
63
+ - **Range-based paging**: `page`, `load_some` and `rebuild_field_index` read a page of entities with a single ordered `range` call on the stable map (Basilisk `StableBTreeMap.range`), so cost tracks the rows returned rather than the highest id ever issued, and deleted ids are free. Backends without `range` (older CDK builds, plain dict test doubles) transparently fall back to one read per id. `instances()` and `find()` load a whole type: they are deprecated and raise `TooLargeError` past 1,000 entities (use `count()`, `find_by()` or `page()`).
64
64
  - **Entity-Relational Database**: Create, read and write entities with OneToOne, OneToMany, ManyToOne, and ManyToMany relationships.
65
65
  - **Schema Versioning & Upgrade Safety**: Automatic schema introspection, compatibility checking, and auto-migration for safe changes. Breaking changes without a `migrate()` method are rejected, preventing data corruption on canister upgrades.
66
66
  - **Entity Hooks**: Intercept and control entity lifecycle events (create, modify, delete) with `on_event` hooks.
@@ -163,11 +163,12 @@ Then use the defined entities to store objects:
163
163
 
164
164
  assert Person.count() == 3
165
165
  assert Person.max_id() == 4
166
- assert Person.instances() == [john, alice, eva]
167
166
 
168
- # Cursor-based pagination
169
- assert Person.load_some(0, 2) == [john, alice]
170
- assert Person.load_some(2, 2) == [eva]
167
+ # Cursor-based pagination: (page, next_from_id), None on the last page
168
+ page, cursor = Person.page(1, 2)
169
+ assert page == [john, alice]
170
+ page, cursor = Person.page(cursor, 2)
171
+ assert page == [eva] and cursor is None
171
172
 
172
173
  # Retrieve database contents in JSON format
173
174
  print(Database.get_instance().dump_json(pretty=True))
@@ -229,7 +230,29 @@ while cursor is not None:
229
230
 
230
231
  The rebuild is idempotent. The `indexed` flag is not part of the schema descriptor, so toggling it never triggers upgrade-compatibility errors.
231
232
 
232
- Note: each distinct value's ID list is stored in a single storage entry, so extremely hot values are bounded by the backing store's `max_value_size` — the same constraint as relationship reverse indexes.
233
+ Each matching id is its own storage entry, so a value can match any number of entities without reaching the backing store's `max_value_size`.
234
+
235
+ ## Large Relations
236
+
237
+ `OneToMany` and `ManyToMany` getters return a `RelationView`, not a list. Questions about the relation and single pages never load every member, so they stay cheap however large it grows:
238
+
239
+ ```python
240
+ if section.stands: ... # one read
241
+ len(section.stands) # one read
242
+ stand in section.stands # two reads
243
+ stands, cursor = section.stands.page(1, 50) # one range read + 50 loads
244
+ group.members.add(user) # index writes only
245
+ ```
246
+
247
+ Iterating, indexing and comparing still work, but they load every member, so past `constants.MAX_UNPAGED` (1,000) they raise `TooLargeError` instead of running into the IC instruction limit. Walk a large relation with `page()`, across calls if needed:
248
+
249
+ ```python
250
+ cursor = 1
251
+ while cursor is not None:
252
+ stands, cursor = section.stands.page(cursor, 50)
253
+ ```
254
+
255
+ `list(view)` gives a plain list. Deleting a parent is O(1): its index entries are purged in bounded steps by later writes, or by `Database.get_instance().purge_step()` from a timer.
233
256
 
234
257
  ## Namespaces
235
258
 
@@ -0,0 +1,19 @@
1
+ ic_python_db/__init__.py,sha256=amMTKKg0eRKGSc6Hz8O-ZqhZfwOZFwRbfxxB5UQfX5o,1083
2
+ ic_python_db/_cdk.py,sha256=zTj8HzXUcPz8R73nh5bo6JMkCorp-KSW1hDutzi5LZs,387
3
+ ic_python_db/constants.py,sha256=gs6klyqUxUjE0LFJxXpaya8QEgYeKx4ovP9Mu04rtHk,351
4
+ ic_python_db/context.py,sha256=97F4_EAJm3adOnUfh0SaDisX3nc7F2vFIZwZUsUAy58,1080
5
+ ic_python_db/db_engine.py,sha256=KniUAn9Gzn8G8bhj-8Kz0-Q1xX6dIvtcN3t3Ng-STSc,36792
6
+ ic_python_db/entity.py,sha256=fRUN0gBBN-KtEXOZOh5rxXlskFltfG8kDlwq1fK6Ck0,54408
7
+ ic_python_db/errors.py,sha256=q_2oAu2mX4B3eGySS-8xOU_CIoKoPlxKwECkbxaHQ1E,366
8
+ ic_python_db/hooks.py,sha256=B1wV18dAMH-PootSripmMpG9Cr9M3W3lcNEF7QvPrL4,1620
9
+ ic_python_db/mixins.py,sha256=qZmb3ssXYf2NSjHJtgmdsOlkIJdouDjPHdQ9tZmrZi8,1937
10
+ ic_python_db/properties.py,sha256=yo2mPH2OjzYvGM3KKSAaiIGTep_qv2496ZT9-_T34FU,31253
11
+ ic_python_db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ ic_python_db/schema.py,sha256=h1RAbapExo8pMD5IQvK8ZTVkgyonbPV9qSqliSP0Y14,16249
13
+ ic_python_db/storage.py,sha256=899qyuA3alpAFIPM23A_07vqquUB5ZG5GjjpzlHW1Os,3828
14
+ ic_python_db/system_time.py,sha256=xrx8NfjLXCGsac3hUpuvsNwCH91tQ9jfH4N-XFo8iJk,2614
15
+ ic_python_db-0.13.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
16
+ ic_python_db-0.13.0.dist-info/METADATA,sha256=lwTLW2L4sj_aLQ--SxXjAFZ0WBRb7of1UXi7YqCa83I,18319
17
+ ic_python_db-0.13.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
18
+ ic_python_db-0.13.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
19
+ ic_python_db-0.13.0.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (84.0.0)
2
+ Generator: setuptools (84.2.1)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,18 +0,0 @@
1
- ic_python_db/__init__.py,sha256=8F0p76Ez1Hhcb9iHKzJjrCWp4aZGgvAGdP9-b9Ka-CU,990
2
- ic_python_db/_cdk.py,sha256=zTj8HzXUcPz8R73nh5bo6JMkCorp-KSW1hDutzi5LZs,387
3
- ic_python_db/constants.py,sha256=oLvKJev-65gXFd284DdYutWBX95sTF5Xkikc6IC4mhM,118
4
- ic_python_db/context.py,sha256=97F4_EAJm3adOnUfh0SaDisX3nc7F2vFIZwZUsUAy58,1080
5
- ic_python_db/db_engine.py,sha256=wK6mJshJMjrmVa7LPjA5ZVz1LXSxTtRjht6MU2mDM6k,26133
6
- ic_python_db/entity.py,sha256=zU_7zoYUs125DToM4Htywc9fhz5NxZhbdvYGMei5RJs,50437
7
- ic_python_db/hooks.py,sha256=B1wV18dAMH-PootSripmMpG9Cr9M3W3lcNEF7QvPrL4,1620
8
- ic_python_db/mixins.py,sha256=qZmb3ssXYf2NSjHJtgmdsOlkIJdouDjPHdQ9tZmrZi8,1937
9
- ic_python_db/properties.py,sha256=l7q3Iv8R-57XMlYwz2gcJV9mObYV_LfR6mw2jZgGwns,27556
10
- ic_python_db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
- ic_python_db/schema.py,sha256=h1RAbapExo8pMD5IQvK8ZTVkgyonbPV9qSqliSP0Y14,16249
12
- ic_python_db/storage.py,sha256=899qyuA3alpAFIPM23A_07vqquUB5ZG5GjjpzlHW1Os,3828
13
- ic_python_db/system_time.py,sha256=xrx8NfjLXCGsac3hUpuvsNwCH91tQ9jfH4N-XFo8iJk,2614
14
- ic_python_db-0.12.0.dist-info/licenses/LICENSE,sha256=6q6XYNOGnJcVSus2bAezFn7bU_2Y5T6W4aGQHBb8X-c,1079
15
- ic_python_db-0.12.0.dist-info/METADATA,sha256=9AjuP2o844gKy9w2Ee-vSZNrqBrEglF6GvsE3v-JHcI,17115
16
- ic_python_db-0.12.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
17
- ic_python_db-0.12.0.dist-info/top_level.txt,sha256=ZaOTqhWKtJQEuXmqR920AqbDBAqv1mvsKj3TClWimDk,13
18
- ic_python_db-0.12.0.dist-info/RECORD,,