event-reduce 7.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,89 @@
1
+ """
2
+ EventReduce: update live query results from change events without re-running the query.
3
+
4
+ Only the BDD that the JavaScript implementation generated is used, it is not created or optimized here.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, TypeVar
10
+
11
+ from .actions import ACTION_FUNCTION_LIST, ORDERED_ACTION_LIST, ActionFunction
12
+ from .bdd import resolve_input
13
+ from .bdd_generated import PACKED_BDD
14
+ from .result_window_buffer import (
15
+ ResultWindowBuffer,
16
+ ResultWindowBufferChangeResult,
17
+ ResultWindowBufferQuery,
18
+ ResultWindowBufferResult,
19
+ get_recommended_result_window_buffer_size,
20
+ )
21
+ from .states import ORDERED_STATE_LIST, get_state_set, resolve_state
22
+ from .types import (
23
+ ActionName,
24
+ ChangeEvent,
25
+ EventReduceError,
26
+ QueryParams,
27
+ ResultKeyDocumentMap,
28
+ StateName,
29
+ StateResolveFunctionInput,
30
+ WriteOperation,
31
+ )
32
+ from .util import dict_query_params, get_property
33
+
34
+ D = TypeVar("D")
35
+
36
+
37
+ def calculate_action_name(input: StateResolveFunctionInput[Any]) -> ActionName:
38
+ """
39
+ Returns the action that transforms the previous results into the new results.
40
+ If it is RUN_FULL_QUERY_AGAIN, the query must be run over the database again.
41
+ """
42
+ return ORDERED_ACTION_LIST[resolve_input(input)]
43
+
44
+
45
+ def run_action(
46
+ action: ActionName,
47
+ query_params: QueryParams[D],
48
+ change_event: ChangeEvent[D],
49
+ previous_results: list[D],
50
+ key_document_map: dict[str, D] | None = None,
51
+ ) -> list[D]:
52
+ """
53
+ Applies the action to the results.
54
+ For performance reasons, it mutates previous_results and the key_document_map
55
+ and returns the mutated results.
56
+ Raises an EventReduceError for RUN_FULL_QUERY_AGAIN and UNKNOWN_ACTION.
57
+ """
58
+ ACTION_FUNCTION_LIST[ORDERED_ACTION_LIST.index(action)](
59
+ StateResolveFunctionInput(query_params, change_event, previous_results, key_document_map)
60
+ )
61
+ return previous_results
62
+
63
+
64
+ __all__ = [
65
+ "ORDERED_ACTION_LIST",
66
+ "ORDERED_STATE_LIST",
67
+ "PACKED_BDD",
68
+ "ActionFunction",
69
+ "ActionName",
70
+ "ChangeEvent",
71
+ "EventReduceError",
72
+ "QueryParams",
73
+ "ResultKeyDocumentMap",
74
+ "ResultWindowBuffer",
75
+ "ResultWindowBufferChangeResult",
76
+ "ResultWindowBufferQuery",
77
+ "ResultWindowBufferResult",
78
+ "StateName",
79
+ "StateResolveFunctionInput",
80
+ "WriteOperation",
81
+ "calculate_action_name",
82
+ "dict_query_params",
83
+ "get_property",
84
+ "get_recommended_result_window_buffer_size",
85
+ "get_state_set",
86
+ "resolve_input",
87
+ "resolve_state",
88
+ "run_action",
89
+ ]
@@ -0,0 +1,183 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from typing import Any
5
+
6
+ from .types import ActionName, EventReduceError, StateResolveFunctionInput
7
+ from .util import index_of_document_id, push_at_sort_position
8
+
9
+ ActionFunction = Callable[[StateResolveFunctionInput[Any]], None]
10
+ """For performance reasons, action functions mutate the results and the key document map of the input."""
11
+
12
+ ORDERED_ACTION_LIST: list[ActionName] = list(ActionName)
13
+ """All actions, in the same order as the JavaScript implementation. The BDD references the actions by their index."""
14
+
15
+
16
+ def _find_index_of_changed_doc(input: StateResolveFunctionInput[Any]) -> int:
17
+ """Returns the index of the changed document inside of the previous results or -1."""
18
+ id = input.change_event.id
19
+ if input.key_document_map is not None and id not in input.key_document_map:
20
+ return -1
21
+ return index_of_document_id(input.query_params, input.previous_results, id)
22
+
23
+
24
+ def _changed_doc(input: StateResolveFunctionInput[Any]) -> Any:
25
+ """The new document of the change event. Only the BDD knows that it exists for the given action."""
26
+ doc = input.change_event.doc
27
+ if doc is None:
28
+ raise EventReduceError("UNK_ACT")
29
+ return doc
30
+
31
+
32
+ def _add_to_map(input: StateResolveFunctionInput[Any], doc: Any) -> None:
33
+ if input.key_document_map is not None:
34
+ input.key_document_map[input.change_event.id] = doc
35
+
36
+
37
+ def _remove_from_map(input: StateResolveFunctionInput[Any], id: str) -> None:
38
+ if input.key_document_map is not None:
39
+ input.key_document_map.pop(id, None)
40
+
41
+
42
+ def do_nothing(input: StateResolveFunctionInput[Any]) -> None:
43
+ pass
44
+
45
+
46
+ def insert_first(input: StateResolveFunctionInput[Any]) -> None:
47
+ doc = _changed_doc(input)
48
+ input.previous_results.insert(0, doc)
49
+ _add_to_map(input, doc)
50
+
51
+
52
+ def insert_last(input: StateResolveFunctionInput[Any]) -> None:
53
+ doc = _changed_doc(input)
54
+ input.previous_results.append(doc)
55
+ _add_to_map(input, doc)
56
+
57
+
58
+ def remove_first_item(input: StateResolveFunctionInput[Any]) -> None:
59
+ if not input.previous_results:
60
+ return
61
+ first = input.previous_results.pop(0)
62
+ _remove_from_map(input, input.query_params.get_document_id(first))
63
+
64
+
65
+ def remove_last_item(input: StateResolveFunctionInput[Any]) -> None:
66
+ if not input.previous_results:
67
+ return
68
+ last = input.previous_results.pop()
69
+ _remove_from_map(input, input.query_params.get_document_id(last))
70
+
71
+
72
+ def remove_first_insert_last(input: StateResolveFunctionInput[Any]) -> None:
73
+ remove_first_item(input)
74
+ insert_last(input)
75
+
76
+
77
+ def remove_last_insert_first(input: StateResolveFunctionInput[Any]) -> None:
78
+ remove_last_item(input)
79
+ insert_first(input)
80
+
81
+
82
+ def remove_first_insert_first(input: StateResolveFunctionInput[Any]) -> None:
83
+ remove_first_item(input)
84
+ insert_first(input)
85
+
86
+
87
+ def remove_last_insert_last(input: StateResolveFunctionInput[Any]) -> None:
88
+ remove_last_item(input)
89
+ insert_last(input)
90
+
91
+
92
+ def remove_existing(input: StateResolveFunctionInput[Any]) -> None:
93
+ index = _find_index_of_changed_doc(input)
94
+ _remove_from_map(input, input.change_event.id)
95
+ if index >= 0:
96
+ del input.previous_results[index]
97
+
98
+
99
+ def replace_existing(input: StateResolveFunctionInput[Any]) -> None:
100
+ index = _find_index_of_changed_doc(input)
101
+ if index >= 0:
102
+ doc = _changed_doc(input)
103
+ input.previous_results[index] = doc
104
+ _add_to_map(input, doc)
105
+
106
+
107
+ def insert_at_sort_position(input: StateResolveFunctionInput[Any]) -> None:
108
+ doc = _changed_doc(input)
109
+ id = input.change_event.id
110
+ key_document_map = input.key_document_map
111
+ # If document is already in results,
112
+ # we cannot add it again because it would throw on non-deterministic ordering.
113
+ if key_document_map is not None:
114
+ if id in key_document_map:
115
+ return
116
+ key_document_map[id] = doc
117
+ elif index_of_document_id(input.query_params, input.previous_results, id) >= 0:
118
+ return
119
+ push_at_sort_position(input.previous_results, doc, input.query_params.sort_comparator)
120
+
121
+
122
+ def remove_existing_and_insert_at_sort_position(input: StateResolveFunctionInput[Any]) -> None:
123
+ remove_existing(input)
124
+ insert_at_sort_position(input)
125
+
126
+
127
+ def _insert_at_sort_position_and_remove(input: StateResolveFunctionInput[Any], remove: ActionFunction) -> None:
128
+ length_before = len(input.previous_results)
129
+ insert_at_sort_position(input)
130
+ # insert_at_sort_position() does nothing if the document is already in the results
131
+ if len(input.previous_results) > length_before:
132
+ remove(input)
133
+
134
+
135
+ def insert_at_sort_position_remove_last(input: StateResolveFunctionInput[Any]) -> None:
136
+ """
137
+ Inserts the document at its sort position and then removes the last document.
138
+ Used when the limit was reached and the new document
139
+ sorts in between the first and the last document,
140
+ so the previous last document is pushed out of the results.
141
+ """
142
+ _insert_at_sort_position_and_remove(input, remove_last_item)
143
+
144
+
145
+ def insert_at_sort_position_remove_first(input: StateResolveFunctionInput[Any]) -> None:
146
+ """
147
+ Inserts the document at its sort position and then removes the first document.
148
+ Used when a document moves out of the skipped documents into the results,
149
+ so the previous first document is pushed into the skipped documents.
150
+ """
151
+ _insert_at_sort_position_and_remove(input, remove_first_item)
152
+
153
+
154
+ def run_full_query_again(input: StateResolveFunctionInput[Any]) -> None:
155
+ raise EventReduceError("RUN_FULL")
156
+
157
+
158
+ def unknown_action(input: StateResolveFunctionInput[Any]) -> None:
159
+ raise EventReduceError("UNK_ACT")
160
+
161
+
162
+ ACTION_FUNCTION_LIST: list[ActionFunction] = [
163
+ do_nothing,
164
+ insert_first,
165
+ insert_last,
166
+ remove_first_item,
167
+ remove_last_item,
168
+ remove_first_insert_last,
169
+ remove_last_insert_first,
170
+ remove_first_insert_first,
171
+ remove_last_insert_last,
172
+ remove_existing,
173
+ replace_existing,
174
+ # alwaysWrong is only used by the truth-table generator and never returned by the BDD
175
+ unknown_action,
176
+ insert_at_sort_position,
177
+ remove_existing_and_insert_at_sort_position,
178
+ insert_at_sort_position_remove_last,
179
+ insert_at_sort_position_remove_first,
180
+ run_full_query_again,
181
+ unknown_action,
182
+ ]
183
+ """The action functions in the same order as ORDERED_ACTION_LIST."""
event_reduce/bdd.py ADDED
@@ -0,0 +1,38 @@
1
+ """
2
+ Resolves the action with the BDD of the JavaScript implementation.
3
+ Only the already generated BDD is used, it is not created or optimized here.
4
+
5
+ The BDD is a PackedBdd of the binary-decision-diagram npm package,
6
+ bdd_generated.PACKED_BDD holds its decoded entries.
7
+
8
+ Layout of the entries:
9
+ - Every non-leaf node takes three consecutive entries: [level, ref0, ref1]
10
+ - level is the index of the state function in ORDERED_STATE_LIST
11
+ - ref0 is followed when the state is false, ref1 when it is true
12
+ - The node with the id x starts at index 3 * x, the root node has the id 0.
13
+ - With n = len(entries) / 3, a reference r < n points to the node with the id r,
14
+ r >= n is a leaf with the action index r - n in ORDERED_ACTION_LIST.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from typing import Any
20
+
21
+ from .bdd_generated import PACKED_BDD
22
+ from .states import STATE_RESOLVE_FUNCTION_BY_INDEX
23
+ from .types import StateResolveFunctionInput
24
+
25
+ _NODE_COUNT = len(PACKED_BDD) // 3
26
+
27
+
28
+ def resolve_input(input: StateResolveFunctionInput[Any]) -> int:
29
+ """Walks the BDD and returns the index of the action in ORDERED_ACTION_LIST."""
30
+ bdd = PACKED_BDD
31
+ fns = STATE_RESOLVE_FUNCTION_BY_INDEX
32
+ n = _NODE_COUNT
33
+ i = 0
34
+ while True:
35
+ ref = bdd[i + 2] if fns[bdd[i]](input) else bdd[i + 1]
36
+ if ref >= n:
37
+ return ref - n
38
+ i = ref * 3
@@ -0,0 +1,4 @@
1
+ # Code generated by the truth-table-generator of the JavaScript implementation. DO NOT EDIT.
2
+ # Run `npm run optimize-bdd` in ./javascript instead.
3
+
4
+ PACKED_BDD: tuple[int, ...] = (1, 1, 2, 0, 3, 4, 17, 5, 6, 6, 7, 92, 17, 92, 8, 6, 9, 92, 6, 10, 11, 11, 12, 13, 6, 14, 15, 11, 16, 13, 11, 17, 18, 5, 93, 19, 5, 92, 20, 7, 21, 108, 5, 22, 23, 5, 93, 108, 5, 92, 24, 5, 22, 25, 5, 26, 27, 16, 108, 92, 12, 92, 28, 8, 29, 95, 7, 30, 31, 7, 32, 33, 10, 34, 20, 10, 35, 36, 10, 37, 38, 10, 37, 39, 7, 40, 41, 9, 101, 96, 15, 42, 94, 15, 43, 92, 15, 44, 94, 15, 45, 92, 7, 46, 47, 7, 48, 49, 12, 23, 50, 8, 51, 52, 7, 53, 54, 7, 55, 56, 16, 92, 95, 16, 92, 108, 14, 104, 93, 4, 57, 97, 14, 104, 108, 4, 58, 108, 15, 59, 92, 15, 60, 92, 15, 61, 94, 15, 62, 92, 7, 63, 64, 9, 102, 100, 9, 99, 97, 8, 65, 66, 8, 67, 68, 8, 69, 70, 8, 71, 72, 14, 106, 98, 14, 106, 108, 14, 92, 40, 4, 73, 41, 14, 104, 19, 4, 74, 19, 15, 75, 76, 15, 77, 41, 9, 105, 78, 9, 79, 97, 9, 80, 81, 9, 82, 108, 9, 83, 84, 9, 85, 108, 15, 86, 108, 9, 71, 108, 14, 92, 41, 14, 106, 19, 14, 87, 19, 16, 94, 97, 4, 88, 19, 15, 89, 100, 15, 90, 97, 15, 105, 108, 15, 89, 108, 15, 90, 108, 15, 86, 105, 15, 86, 100, 15, 86, 97, 14, 105, 108, 16, 104, 107, 14, 91, 19, 14, 105, 98, 14, 105, 99, 16, 106, 107) # fmt: skip
event_reduce/py.typed ADDED
File without changes
@@ -0,0 +1,356 @@
1
+ """
2
+ Result window buffer
3
+ ------------
4
+ For a query with a limit or a skip, EventReduce returns runFullQueryAgain
5
+ when a row has to move into the result window that is not in the previous results:
6
+ the row after the last row (a row left a full window) or the last skipped row
7
+ (a row was added in front of the window).
8
+ The result window buffer keeps rows on both sides of the window in memory,
9
+ so these rows can be taken from memory instead.
10
+ See docs/result-window-buffer.md for the explanation and measurements.
11
+
12
+ Invariant: `buffer` is always an exact, contiguous slice of the full sorted result
13
+ (without skip and limit): exactly `offset` matching rows sort before its first row,
14
+ and it never contains a wrong row or misses a row in between.
15
+ If `complete` is true, there are no rows after its last row.
16
+ Only if the buffer is empty, `offset` can be larger than the real number of rows.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import math
22
+ from collections.abc import Iterable
23
+ from dataclasses import dataclass, field
24
+ from enum import Enum
25
+ from typing import Generic, NamedTuple, TypeVar
26
+
27
+ from . import actions
28
+ from .actions import ACTION_FUNCTION_LIST, ActionFunction
29
+ from .bdd import resolve_input
30
+ from .types import ChangeEvent, EventReduceError, QueryParams, StateResolveFunctionInput
31
+ from .util import index_of_document_id, push_at_sort_position
32
+
33
+ D = TypeVar("D")
34
+
35
+ BufferSize = int | float
36
+ """A whole number of rows, or math.inf to keep all rows on that side."""
37
+
38
+
39
+ class ResultWindowBufferResult(str, Enum):
40
+ """What apply_change_event() did."""
41
+
42
+ EVENT_REDUCE = "eventReduce"
43
+ """An EventReduce action calculated the new buffer."""
44
+ BUFFER = "buffer"
45
+ """EventReduce could not do it (or would have dropped a row), the slice rule did it."""
46
+ RUN_FULL_QUERY_AGAIN = "runFullQueryAgain"
47
+ """The caller must run the query with the skip and limit of query() and pass the rows to reset()."""
48
+
49
+
50
+ class ResultWindowBufferQuery(NamedTuple):
51
+ """The skip and limit to run the query of the buffer with, limit is None if all rows after the skip are needed."""
52
+
53
+ skip: int
54
+ limit: int | None
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ class ResultWindowBufferChangeResult(Generic[D]):
59
+ """
60
+ What apply_change_events() returns.
61
+ results are the rows of the original skip and limit,
62
+ changed is true if they are not the same rows as before the change events.
63
+ """
64
+
65
+ run_full_query_again: bool
66
+ changed: bool = False
67
+ results: list[D] = field(default_factory=list)
68
+
69
+
70
+ ACTIONS_THAT_DROP_A_ROW: frozenset[ActionFunction] = frozenset(
71
+ {
72
+ actions.remove_first_item,
73
+ actions.remove_last_item,
74
+ actions.remove_first_insert_last,
75
+ actions.remove_last_insert_first,
76
+ actions.remove_first_insert_first,
77
+ actions.remove_last_insert_last,
78
+ actions.insert_at_sort_position_remove_last,
79
+ actions.insert_at_sort_position_remove_first,
80
+ }
81
+ )
82
+ """Actions that keep the length of the result by dropping its first or last row. The slice rule can keep that row."""
83
+
84
+
85
+ def get_recommended_result_window_buffer_size(limit: int | None = None) -> int:
86
+ """
87
+ The recommended buffer size for each side of the result window.
88
+ About log2(limit) for small limits, grows faster around limit 400 and is at most 30.
89
+ Returns 0 for a limit of 1 (like a find-one query) and for queries without a limit.
90
+ """
91
+ if not limit:
92
+ return 0
93
+ return min(30, math.floor(math.log2(limit) + 19 / (1 + (400 / limit) ** 2)))
94
+
95
+
96
+ def _to_buffer_size(size: BufferSize | None) -> BufferSize:
97
+ """
98
+ Buffer sizes are whole numbers, rounded down. Not set means 0,
99
+ math.inf keeps all rows on that side.
100
+ Raises on negative sizes and on values that are not numbers, like NaN.
101
+ """
102
+ if size is None:
103
+ return 0
104
+ if isinstance(size, bool) or not isinstance(size, (int, float)) or math.isnan(size) or size < 0:
105
+ raise EventReduceError("RWB_SIZE")
106
+ return size if math.isinf(size) else math.floor(size)
107
+
108
+
109
+ class ResultWindowBuffer(Generic[D]):
110
+ """
111
+ Keeps rows on both sides of the result window of a query with a skip or limit in memory.
112
+ Create it, run the query with the skip and limit of query()
113
+ and pass the rows to reset() before using it.
114
+ """
115
+
116
+ def __init__(
117
+ self,
118
+ query_params: QueryParams[D],
119
+ limit_buffer_size: BufferSize | None = None,
120
+ skip_buffer_size: BufferSize | None = None,
121
+ ) -> None:
122
+ self.query_params = query_params
123
+ """QueryParams of the original query, with the skip and limit that are shown to the user."""
124
+ limit_size = _to_buffer_size(limit_buffer_size)
125
+ skip_size = _to_buffer_size(skip_buffer_size)
126
+ # with limit 0 the result is always empty, so no rows are kept
127
+ is_empty_query = query_params.limit == 0
128
+ self.limit_buffer_size: BufferSize = 0 if is_empty_query else limit_size
129
+ """Amount of rows to keep after the last row of the result window."""
130
+ self.skip_buffer_size: BufferSize = 0 if is_empty_query else skip_size
131
+ """Amount of rows to keep before the first row of the result window, out of the skipped rows."""
132
+ self.buffer: list[D] = []
133
+ self.key_document_map: dict[str, D] = {}
134
+ self.offset = 0
135
+ """Number of matching rows that sort before the first row of the buffer."""
136
+ self.complete = False
137
+ """True if there are no rows after the last row of the buffer."""
138
+
139
+ def _skip(self) -> int:
140
+ return self.query_params.skip or 0
141
+
142
+ def query(self) -> ResultWindowBufferQuery:
143
+ """The skip and limit to run the query of the buffer with."""
144
+ skip = self._skip()
145
+ buffer_skip = max(0, skip - self.skip_buffer_size)
146
+ limit = self.query_params.limit
147
+ buffer_limit = None
148
+ if limit is not None and not math.isinf(self.limit_buffer_size):
149
+ buffer_limit = int((skip - buffer_skip) + limit + self.limit_buffer_size)
150
+ return ResultWindowBufferQuery(int(buffer_skip), buffer_limit)
151
+
152
+ def reset(self, rows: Iterable[D]) -> None:
153
+ """Set the buffer to the rows of a fresh query that ran with the skip and limit of query()."""
154
+ query = self.query()
155
+ get_id = self.query_params.get_document_id
156
+ self.buffer = list(rows)
157
+ self.key_document_map = {get_id(doc): doc for doc in self.buffer}
158
+ self.offset = query.skip
159
+ # fewer rows than requested means there are no more rows
160
+ self.complete = query.limit is None or len(self.buffer) < query.limit
161
+
162
+ def results(self) -> list[D]:
163
+ """
164
+ The result that is shown to the user.
165
+ Raises if the result window is not in the buffer, which happens
166
+ before reset() was called or when apply_change_event() returned RUN_FULL_QUERY_AGAIN
167
+ and the query did not run again.
168
+ """
169
+ if not self._is_result_window_in_buffer():
170
+ raise EventReduceError("RWB_OUTDATED")
171
+ start = self._skip() - self.offset
172
+ limit = self.query_params.limit
173
+ return self.buffer[start : None if limit is None else start + limit]
174
+
175
+ def apply_change_event(self, change_event: ChangeEvent[D]) -> ResultWindowBufferResult:
176
+ """Updates the buffer with the change event."""
177
+ buffer = self.buffer
178
+ if self.query_params.limit == 0:
179
+ # the result of a query with limit 0 is always empty, nothing can change it
180
+ return ResultWindowBufferResult.EVENT_REDUCE
181
+ if not self.complete and not buffer:
182
+ return ResultWindowBufferResult.RUN_FULL_QUERY_AGAIN
183
+
184
+ # 1. Let EventReduce handle the buffer as if it was the result of a query
185
+ # with skip = offset and limit = len(buffer). This is exactly true,
186
+ # because the buffer is an exact slice of the full result.
187
+ # If the buffer is complete, we use len(buffer) + 1, so EventReduce
188
+ # knows that the limit is not reached and there are no more rows.
189
+ query_params = self.query_params
190
+ input = StateResolveFunctionInput(
191
+ QueryParams(
192
+ query_params.get_document_id,
193
+ query_params.query_matcher,
194
+ query_params.sort_comparator,
195
+ query_params.sort_fields_changed,
196
+ self.offset if self.offset > 0 else None,
197
+ len(buffer) + 1 if self.complete else len(buffer),
198
+ ),
199
+ change_event,
200
+ buffer,
201
+ self.key_document_map,
202
+ )
203
+ action = ACTION_FUNCTION_LIST[resolve_input(input)]
204
+ if action is actions.run_full_query_again or action in ACTIONS_THAT_DROP_A_ROW:
205
+ # 2. EventReduce needs a row before or after the buffer, which it does not know,
206
+ # or it would drop a row to keep the length of the result.
207
+ # But the buffer does not have to keep its length, only to stay an exact slice,
208
+ # so the slice rule lets it shrink or grow instead.
209
+ self._apply_slice_rule(change_event)
210
+ result = ResultWindowBufferResult.BUFFER
211
+ else:
212
+ action(input)
213
+ result = ResultWindowBufferResult.EVENT_REDUCE
214
+
215
+ # 3. Do not let the buffer grow without bounds on either side.
216
+ self._trim()
217
+
218
+ # 4. Run the query again if the result window does not fit into the buffer anymore.
219
+ if not self._is_result_window_in_buffer():
220
+ return ResultWindowBufferResult.RUN_FULL_QUERY_AGAIN
221
+ return result
222
+
223
+ def apply_change_events(self, change_events: Iterable[ChangeEvent[D]]) -> ResultWindowBufferChangeResult[D]:
224
+ """
225
+ Updates the buffer with all change events.
226
+ Returns the new results, and if they changed,
227
+ or run_full_query_again if the result window does not fit into the buffer anymore.
228
+ Then run the query of query() and pass the rows to reset().
229
+ """
230
+ if not self._is_result_window_in_buffer():
231
+ return ResultWindowBufferChangeResult(run_full_query_again=True)
232
+ previous_results = self.results()
233
+ for change_event in change_events:
234
+ if self.apply_change_event(change_event) == ResultWindowBufferResult.RUN_FULL_QUERY_AGAIN:
235
+ return ResultWindowBufferChangeResult(run_full_query_again=True)
236
+ results = self.results()
237
+ # Changed rows are always new objects from the change events,
238
+ # so comparing the rows by reference finds every change.
239
+ changed = len(results) != len(previous_results) or any(
240
+ row is not previous for row, previous in zip(results, previous_results, strict=False)
241
+ )
242
+ return ResultWindowBufferChangeResult(run_full_query_again=False, changed=changed, results=results)
243
+
244
+ def _is_result_window_in_buffer(self) -> bool:
245
+ """True if all rows of the result window are in the buffer."""
246
+ start = self._skip() - self.offset
247
+ if start < 0:
248
+ return False
249
+ if self.complete:
250
+ return True
251
+ limit = self.query_params.limit
252
+ return limit is not None and len(self.buffer) >= start + limit
253
+
254
+ def _trim(self) -> None:
255
+ """
256
+ Removes rows before the window that are more than skip_buffer_size
257
+ and rows after the window that are more than limit_buffer_size.
258
+ """
259
+ get_id = self.query_params.get_document_id
260
+ rows_before_window = self._skip() - self.offset
261
+ remove_from_start = min(rows_before_window - self.skip_buffer_size, len(self.buffer))
262
+ if remove_from_start > 0:
263
+ remove = int(remove_from_start)
264
+ for doc in self.buffer[:remove]:
265
+ del self.key_document_map[get_id(doc)]
266
+ del self.buffer[:remove]
267
+ self.offset += remove
268
+
269
+ limit = self.query_params.limit
270
+ if limit is not None:
271
+ max_length = max(0, self._skip() - self.offset) + limit + self.limit_buffer_size
272
+ if len(self.buffer) > max_length:
273
+ keep = int(max_length)
274
+ for doc in self.buffer[keep:]:
275
+ del self.key_document_map[get_id(doc)]
276
+ del self.buffer[keep:]
277
+ self.complete = False
278
+
279
+ def _apply_slice_rule(self, change_event: ChangeEvent[D]) -> None:
280
+ """
281
+ The slice rule:
282
+ Exactly `offset` matching rows sort before the first row of the buffer,
283
+ and all matching rows that sort between the first and the last row are in the buffer.
284
+ After the change event, this is still true for the old first and last row, so:
285
+ - the old version of the changed row is removed from the buffer,
286
+ - the new version is added if it sorts between the old first and last row,
287
+ - offset changes by one if the changed row moved into or out of the rows before the first row.
288
+ If offset is 0, there are no rows before the first row, so a row that sorts before it is added.
289
+ If the buffer is complete, there are no rows after the last row, so a row that sorts after it is added.
290
+
291
+ An empty buffer (only valid when complete) knows only the number of rows before it.
292
+ There, offset is only an upper bound, which stays an upper bound with the same update.
293
+ """
294
+ query_params = self.query_params
295
+ buffer = self.buffer
296
+ previous = change_event.previous
297
+ doc = change_event.doc
298
+ was_matching = previous is not None and query_params.query_matcher(previous)
299
+ does_match_now = doc is not None and query_params.query_matcher(doc)
300
+
301
+ if not buffer:
302
+ if self.offset == 0:
303
+ # there are no rows at all
304
+ if doc is not None and does_match_now:
305
+ self._insert_at_sort_position(doc)
306
+ else:
307
+ self.offset += int(does_match_now) - int(was_matching)
308
+ return
309
+
310
+ first = buffer[0]
311
+ last = buffer[-1]
312
+ has_rows_before = self.offset > 0
313
+ was_before = previous is not None and was_matching and has_rows_before and self._compare(previous, first) < 0
314
+ is_before_now = doc is not None and does_match_now and has_rows_before and self._compare(doc, first) < 0
315
+
316
+ id = change_event.id
317
+ if self.key_document_map.pop(id, None) is not None:
318
+ del buffer[index_of_document_id(query_params, buffer, id)]
319
+ if (
320
+ doc is not None
321
+ and does_match_now
322
+ and not is_before_now
323
+ and (self.complete or self._compare(doc, last) <= 0)
324
+ ):
325
+ self._insert_at_sort_position(doc)
326
+ self.offset += int(is_before_now) - int(was_before)
327
+
328
+ def _compare(self, a: D, b: D) -> int:
329
+ """
330
+ Compares two rows with the sort comparator of the query.
331
+ Returns 0 only for two versions of the same row that sort at the same position.
332
+ For different rows that the comparator sees as equal,
333
+ the primary key decides, so that the order is always deterministic.
334
+ """
335
+ query_params = self.query_params
336
+ id_a = query_params.get_document_id(a)
337
+ id_b = query_params.get_document_id(b)
338
+ result = _sign(query_params.sort_comparator(a, b))
339
+ if id_a == id_b:
340
+ # A deterministic comparator breaks ties by the primary key,
341
+ # which gives the same answer for both directions when the sort values are equal.
342
+ if result == 0 or result == _sign(query_params.sort_comparator(b, a)):
343
+ return 0
344
+ return result
345
+ if result != 0:
346
+ return result
347
+ return -1 if id_a < id_b else 1
348
+
349
+ def _insert_at_sort_position(self, doc: D) -> None:
350
+ # _compare() returns 0 only for another version of the same row, which was removed before
351
+ push_at_sort_position(self.buffer, doc, self._compare)
352
+ self.key_document_map[self.query_params.get_document_id(doc)] = doc
353
+
354
+
355
+ def _sign(value: int) -> int:
356
+ return (value > 0) - (value < 0)
event_reduce/states.py ADDED
@@ -0,0 +1,184 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from typing import Any
5
+
6
+ from .types import StateName, StateResolveFunctionInput, WriteOperation
7
+ from .util import index_of_document_id
8
+
9
+ StateResolveFunction = Callable[[StateResolveFunctionInput[Any]], bool]
10
+
11
+ ORDERED_STATE_LIST: list[StateName] = list(StateName)
12
+ """All states, in the same order as the JavaScript implementation. The BDD references the states by their index."""
13
+
14
+
15
+ def has_limit(input: StateResolveFunctionInput[Any]) -> bool:
16
+ return bool(input.query_params.limit)
17
+
18
+
19
+ def is_find_one(input: StateResolveFunctionInput[Any]) -> bool:
20
+ return input.query_params.limit == 1
21
+
22
+
23
+ def has_skip(input: StateResolveFunctionInput[Any]) -> bool:
24
+ skip = input.query_params.skip
25
+ return skip is not None and skip > 0
26
+
27
+
28
+ def is_delete(input: StateResolveFunctionInput[Any]) -> bool:
29
+ return input.change_event.operation == WriteOperation.DELETE
30
+
31
+
32
+ def is_insert(input: StateResolveFunctionInput[Any]) -> bool:
33
+ return input.change_event.operation == WriteOperation.INSERT
34
+
35
+
36
+ def is_update(input: StateResolveFunctionInput[Any]) -> bool:
37
+ return input.change_event.operation == WriteOperation.UPDATE
38
+
39
+
40
+ def was_limit_reached(input: StateResolveFunctionInput[Any]) -> bool:
41
+ limit = input.query_params.limit
42
+ return bool(limit) and len(input.previous_results) >= limit # type: ignore[operator]
43
+
44
+
45
+ def sort_params_changed(input: StateResolveFunctionInput[Any]) -> bool:
46
+ doc = input.change_event.doc
47
+ prev = input.change_event.previous
48
+ if doc is None:
49
+ return False
50
+ if prev is None:
51
+ return True
52
+ return input.query_params.sort_fields_changed(prev, doc)
53
+
54
+
55
+ def was_in_result(input: StateResolveFunctionInput[Any]) -> bool:
56
+ id = input.change_event.id
57
+ if input.key_document_map is not None:
58
+ return id in input.key_document_map
59
+ return index_of_document_id(input.query_params, input.previous_results, id) >= 0
60
+
61
+
62
+ def was_first(input: StateResolveFunctionInput[Any]) -> bool:
63
+ results = input.previous_results
64
+ return bool(results) and input.query_params.get_document_id(results[0]) == input.change_event.id
65
+
66
+
67
+ def was_last(input: StateResolveFunctionInput[Any]) -> bool:
68
+ results = input.previous_results
69
+ return bool(results) and input.query_params.get_document_id(results[-1]) == input.change_event.id
70
+
71
+
72
+ def _compare_previous_with_result(
73
+ input: StateResolveFunctionInput[Any], first: bool, expected: Callable[[int], bool]
74
+ ) -> bool:
75
+ """
76
+ Compares the previous document with the first or last result item.
77
+ If the changed document is the result item itself,
78
+ we cannot sort-compare them, because it might end in a non-deterministic
79
+ sort order. Because both document could be equal.
80
+ So instead we have to return true.
81
+ """
82
+ prev = input.change_event.previous
83
+ results = input.previous_results
84
+ if prev is None or not results:
85
+ return False
86
+ result_item = results[0] if first else results[-1]
87
+ if input.query_params.get_document_id(result_item) == input.change_event.id:
88
+ return True
89
+ return expected(input.query_params.sort_comparator(prev, result_item))
90
+
91
+
92
+ def _compare_doc_with_result(
93
+ input: StateResolveFunctionInput[Any], first: bool, expected: Callable[[int], bool]
94
+ ) -> bool:
95
+ """
96
+ Compares the new document with the first or last result item.
97
+ If the changed document is the result item itself,
98
+ it compares with the neighbor of the result item instead,
99
+ so we know if the changed document still sorts first or last.
100
+ If there is no neighbor, the changed document is still first or last.
101
+ """
102
+ doc = input.change_event.doc
103
+ results = input.previous_results
104
+ if doc is None or not results:
105
+ return False
106
+ result_item = results[0] if first else results[-1]
107
+ if input.query_params.get_document_id(result_item) == input.change_event.id:
108
+ if len(results) < 2:
109
+ return True
110
+ result_item = results[1] if first else results[-2]
111
+ return expected(input.query_params.sort_comparator(doc, result_item))
112
+
113
+
114
+ def _is_negative(c: int) -> bool:
115
+ return c < 0
116
+
117
+
118
+ def _is_positive(c: int) -> bool:
119
+ return c > 0
120
+
121
+
122
+ def was_sorted_before_first(input: StateResolveFunctionInput[Any]) -> bool:
123
+ return _compare_previous_with_result(input, True, _is_negative)
124
+
125
+
126
+ def was_sorted_after_last(input: StateResolveFunctionInput[Any]) -> bool:
127
+ return _compare_previous_with_result(input, False, _is_positive)
128
+
129
+
130
+ def is_sorted_before_first(input: StateResolveFunctionInput[Any]) -> bool:
131
+ return _compare_doc_with_result(input, True, _is_negative)
132
+
133
+
134
+ def is_sorted_after_last(input: StateResolveFunctionInput[Any]) -> bool:
135
+ return _compare_doc_with_result(input, False, _is_positive)
136
+
137
+
138
+ def was_matching(input: StateResolveFunctionInput[Any]) -> bool:
139
+ prev = input.change_event.previous
140
+ return prev is not None and input.query_params.query_matcher(prev)
141
+
142
+
143
+ def does_match_now(input: StateResolveFunctionInput[Any]) -> bool:
144
+ doc = input.change_event.doc
145
+ return doc is not None and input.query_params.query_matcher(doc)
146
+
147
+
148
+ def was_results_empty(input: StateResolveFunctionInput[Any]) -> bool:
149
+ return not input.previous_results
150
+
151
+
152
+ STATE_RESOLVE_FUNCTION_BY_INDEX: list[StateResolveFunction] = [
153
+ is_insert,
154
+ is_update,
155
+ is_delete,
156
+ has_limit,
157
+ is_find_one,
158
+ has_skip,
159
+ was_results_empty,
160
+ was_limit_reached,
161
+ was_first,
162
+ was_last,
163
+ sort_params_changed,
164
+ was_in_result,
165
+ was_sorted_before_first,
166
+ was_sorted_after_last,
167
+ is_sorted_before_first,
168
+ is_sorted_after_last,
169
+ was_matching,
170
+ does_match_now,
171
+ ]
172
+ """The state functions in the same order as ORDERED_STATE_LIST."""
173
+
174
+
175
+ def resolve_state(state: StateName, input: StateResolveFunctionInput[Any]) -> bool:
176
+ return STATE_RESOLVE_FUNCTION_BY_INDEX[ORDERED_STATE_LIST.index(state)](input)
177
+
178
+
179
+ def get_state_set(input: StateResolveFunctionInput[Any]) -> str:
180
+ """
181
+ Returns the binary representation of all states
182
+ like '010110110111...' where the first '0' means that the first state is false.
183
+ """
184
+ return "".join("1" if fn(input) else "0" for fn in STATE_RESOLVE_FUNCTION_BY_INDEX)
event_reduce/types.py ADDED
@@ -0,0 +1,140 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from dataclasses import dataclass
5
+ from enum import Enum
6
+ from typing import Generic, TypeVar
7
+
8
+ D = TypeVar("D")
9
+
10
+
11
+ class WriteOperation(str, Enum):
12
+ INSERT = "INSERT"
13
+ UPDATE = "UPDATE"
14
+ DELETE = "DELETE"
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class ChangeEvent(Generic[D]):
19
+ """
20
+ A write event on the database.
21
+ `id` is the value of the primary key of the changed document.
22
+ `doc` is the document after the write (None on deletes),
23
+ `previous` the document before the write (None on inserts).
24
+ """
25
+
26
+ operation: WriteOperation
27
+ id: str
28
+ doc: D | None
29
+ previous: D | None
30
+
31
+ @staticmethod
32
+ def insert(id: str, doc: D) -> ChangeEvent[D]:
33
+ return ChangeEvent(WriteOperation.INSERT, id, doc, None)
34
+
35
+ @staticmethod
36
+ def update(id: str, doc: D, previous: D) -> ChangeEvent[D]:
37
+ return ChangeEvent(WriteOperation.UPDATE, id, doc, previous)
38
+
39
+ @staticmethod
40
+ def delete(id: str, previous: D) -> ChangeEvent[D]:
41
+ return ChangeEvent(WriteOperation.DELETE, id, None, previous)
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class QueryParams(Generic[D]):
46
+ """
47
+ Describes the query whose results are kept up to date.
48
+ For dict documents, dict_query_params() creates it from a primary key and the sort fields.
49
+ """
50
+
51
+ get_document_id: Callable[[D], str]
52
+ """Returns the id of a document, must return the same value as the id of the change events."""
53
+ query_matcher: Callable[[D], bool]
54
+ """Returns true if the document matches the query."""
55
+ sort_comparator: Callable[[D, D], int]
56
+ """
57
+ To have a deterministic sorting, the comparator must never return 0
58
+ for two different documents. Use the primary key as last sort field to ensure that.
59
+ """
60
+ sort_fields_changed: Callable[[D, D], bool]
61
+ """
62
+ Returns true if any of the fields that the query sorts by
63
+ differs between the previous and the new version of a document.
64
+ Called as sort_fields_changed(previous, doc).
65
+ """
66
+ skip: int | None = None
67
+ limit: int | None = None
68
+
69
+
70
+ ResultKeyDocumentMap = dict[str, D]
71
+ """Map from the id of each document in the results to the document."""
72
+
73
+
74
+ @dataclass
75
+ class StateResolveFunctionInput(Generic[D]):
76
+ """
77
+ The input of the state functions, of the action functions and of calculate_action_name().
78
+ Action functions mutate previous_results and key_document_map.
79
+ """
80
+
81
+ query_params: QueryParams[D]
82
+ change_event: ChangeEvent[D]
83
+ previous_results: list[D]
84
+ key_document_map: dict[str, D] | None = None
85
+
86
+
87
+ class ActionName(str, Enum):
88
+ """The actions, in the same order as the JavaScript implementation, the values are the JavaScript names."""
89
+
90
+ DO_NOTHING = "doNothing"
91
+ INSERT_FIRST = "insertFirst"
92
+ INSERT_LAST = "insertLast"
93
+ REMOVE_FIRST_ITEM = "removeFirstItem"
94
+ REMOVE_LAST_ITEM = "removeLastItem"
95
+ REMOVE_FIRST_INSERT_LAST = "removeFirstInsertLast"
96
+ REMOVE_LAST_INSERT_FIRST = "removeLastInsertFirst"
97
+ REMOVE_FIRST_INSERT_FIRST = "removeFirstInsertFirst"
98
+ REMOVE_LAST_INSERT_LAST = "removeLastInsertLast"
99
+ REMOVE_EXISTING = "removeExisting"
100
+ REPLACE_EXISTING = "replaceExisting"
101
+ # only used by the truth-table generator, never returned by the BDD
102
+ ALWAYS_WRONG = "alwaysWrong"
103
+ INSERT_AT_SORT_POSITION = "insertAtSortPosition"
104
+ REMOVE_EXISTING_AND_INSERT_AT_SORT_POSITION = "removeExistingAndInsertAtSortPosition"
105
+ INSERT_AT_SORT_POSITION_REMOVE_LAST = "insertAtSortPositionRemoveLast"
106
+ INSERT_AT_SORT_POSITION_REMOVE_FIRST = "insertAtSortPositionRemoveFirst"
107
+ RUN_FULL_QUERY_AGAIN = "runFullQueryAgain"
108
+ # if a state was never reached, we do not know the correct action
109
+ UNKNOWN_ACTION = "unknownAction"
110
+
111
+
112
+ class StateName(str, Enum):
113
+ """The states, in the same order as the JavaScript implementation, the values are the JavaScript names."""
114
+
115
+ IS_INSERT = "isInsert"
116
+ IS_UPDATE = "isUpdate"
117
+ IS_DELETE = "isDelete"
118
+ HAS_LIMIT = "hasLimit"
119
+ IS_FIND_ONE = "isFindOne"
120
+ HAS_SKIP = "hasSkip"
121
+ WAS_RESULTS_EMPTY = "wasResultsEmpty"
122
+ WAS_LIMIT_REACHED = "wasLimitReached"
123
+ WAS_FIRST = "wasFirst"
124
+ WAS_LAST = "wasLast"
125
+ SORT_PARAMS_CHANGED = "sortParamsChanged"
126
+ WAS_IN_RESULT = "wasInResult"
127
+ WAS_SORTED_BEFORE_FIRST = "wasSortedBeforeFirst"
128
+ WAS_SORTED_AFTER_LAST = "wasSortedAfterLast"
129
+ IS_SORTED_BEFORE_FIRST = "isSortedBeforeFirst"
130
+ IS_SORTED_AFTER_LAST = "isSortedAfterLast"
131
+ WAS_MATCHING = "wasMatching"
132
+ DOES_MATCH_NOW = "doesMatchNow"
133
+
134
+
135
+ class EventReduceError(Exception):
136
+ """Raised for the actions that cannot be applied to the results and for invalid input."""
137
+
138
+ def __init__(self, code: str) -> None:
139
+ super().__init__("EventReduce:" + code)
140
+ self.code = code
event_reduce/util.py ADDED
@@ -0,0 +1,110 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable, Mapping, Sequence
4
+ from typing import Any, TypeVar
5
+
6
+ from .types import QueryParams
7
+
8
+ D = TypeVar("D")
9
+
10
+ _MISSING = object()
11
+
12
+
13
+ def get_property(obj: Any, path: str, default: Any = None) -> Any:
14
+ """
15
+ Returns the value at the dot-separated path like 'items.0.rank'.
16
+ List entries are accessed by their index.
17
+ Returns the given default value if the path does not exist.
18
+ """
19
+ return _get_path(obj, path.split("."), default)
20
+
21
+
22
+ def _get_path(obj: Any, keys: list[str], default: Any = None) -> Any:
23
+ for key in keys:
24
+ if isinstance(obj, Mapping):
25
+ obj = obj.get(key, _MISSING)
26
+ elif isinstance(obj, (list, tuple)) and key.isdigit() and int(key) < len(obj):
27
+ obj = obj[int(key)]
28
+ else:
29
+ return default
30
+ if obj is _MISSING or obj is None:
31
+ return default
32
+ return obj
33
+
34
+
35
+ def normalize_sort_field(field: str) -> str:
36
+ """normalizes sort-field, in: '-age', out: 'age'"""
37
+ return field[1:] if field.startswith("-") else field
38
+
39
+
40
+ def dict_query_params(
41
+ primary_key: str,
42
+ sort_fields: Sequence[str],
43
+ query_matcher: Callable[[dict[str, Any]], bool],
44
+ sort_comparator: Callable[[dict[str, Any], dict[str, Any]], int],
45
+ skip: int | None = None,
46
+ limit: int | None = None,
47
+ ) -> QueryParams[dict[str, Any]]:
48
+ """
49
+ QueryParams for dict documents, like the primaryKey and sortFields of the JavaScript implementation.
50
+ The sort fields can be dot-separated paths and can start with '-'.
51
+ """
52
+ # Like JavaScript, fields without a dot are read directly, paths are split only once.
53
+ fields: list[str] = []
54
+ paths: list[list[str]] = []
55
+ for sort_field in sort_fields:
56
+ field = normalize_sort_field(sort_field)
57
+ if "." in field:
58
+ paths.append(field.split("."))
59
+ else:
60
+ fields.append(field)
61
+
62
+ def sort_fields_changed(previous: dict[str, Any], doc: dict[str, Any]) -> bool:
63
+ for field in fields:
64
+ if previous.get(field) != doc.get(field):
65
+ return True
66
+ for path in paths:
67
+ if _get_path(previous, path) != _get_path(doc, path):
68
+ return True
69
+ return False
70
+
71
+ def get_document_id(doc: dict[str, Any]) -> str:
72
+ return str(doc[primary_key])
73
+
74
+ return QueryParams(
75
+ get_document_id=get_document_id,
76
+ query_matcher=query_matcher,
77
+ sort_comparator=sort_comparator,
78
+ sort_fields_changed=sort_fields_changed,
79
+ skip=skip,
80
+ limit=limit,
81
+ )
82
+
83
+
84
+ def index_of_document_id(query_params: QueryParams[D], docs: list[D], id: str) -> int:
85
+ """Returns the index of the document with the given id or -1 if it is not in the given documents."""
86
+ get_id = query_params.get_document_id
87
+ for index, doc in enumerate(docs):
88
+ if get_id(doc) == id:
89
+ return index
90
+ return -1
91
+
92
+
93
+ def push_at_sort_position(items: list[D], item: D, compare: Callable[[D, D], int]) -> None:
94
+ """
95
+ Inserts the item at its sort position, which is much faster than re-sorting the list.
96
+ Items that are equal to the new item stay before it.
97
+ """
98
+ # Shortcut: the item belongs at the end, append() is O(1).
99
+ if not items or compare(items[-1], item) <= 0:
100
+ items.append(item)
101
+ return
102
+ low = 0
103
+ high = len(items)
104
+ while low < high:
105
+ mid = (low + high) >> 1
106
+ if compare(items[mid], item) <= 0:
107
+ low = mid + 1
108
+ else:
109
+ high = mid
110
+ items.insert(low, item)
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.4
2
+ Name: event-reduce
3
+ Version: 7.4.2
4
+ Summary: Update live query results from database change events without re-running the query.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/pubkey/event-reduce
7
+ Project-URL: Repository, https://github.com/pubkey/event-reduce
8
+ Keywords: live-query,reactive,realtime,database,bdd
9
+ Requires-Python: >=3.10
10
+ Description-Content-Type: text/markdown
11
+
12
+ # event-reduce (Python)
13
+
14
+ Python implementation of the [EventReduce algorithm](https://github.com/pubkey/event-reduce).
15
+ It takes the previous results of a query and a change event of the database
16
+ and calculates the new results **in memory, without running the query over the database again**.
17
+
18
+ It uses the BDD that the [JavaScript implementation](../javascript/) created,
19
+ so both always decide on the same action. It needs Python 3.10 or newer and has no dependencies.
20
+
21
+ ## Installation
22
+
23
+ ```sh
24
+ pip install event-reduce
25
+ ```
26
+
27
+ ## Usage
28
+
29
+ ```python
30
+ from event_reduce import (
31
+ ActionName,
32
+ ChangeEvent,
33
+ StateResolveFunctionInput,
34
+ calculate_action_name,
35
+ dict_query_params,
36
+ run_action,
37
+ )
38
+
39
+
40
+ def compare(a, b):
41
+ # must never return 0 for two different documents, so sort by the primary key last
42
+ if a["age"] != b["age"]:
43
+ return -1 if a["age"] < b["age"] else 1
44
+ return -1 if a["id"] < b["id"] else 1
45
+
46
+
47
+ # SELECT * FROM users WHERE age > 18 ORDER BY age, id LIMIT 10
48
+ query_params = dict_query_params(
49
+ primary_key="id",
50
+ sort_fields=["age", "id"],
51
+ query_matcher=lambda doc: doc["age"] > 18,
52
+ sort_comparator=compare,
53
+ limit=10,
54
+ )
55
+
56
+ results = [{"id": "a", "age": 20}, {"id": "b", "age": 30}]
57
+
58
+ # a change event from the change stream of your database
59
+ change_event = ChangeEvent.insert("c", {"id": "c", "age": 25})
60
+
61
+ action = calculate_action_name(StateResolveFunctionInput(query_params, change_event, results))
62
+ if action == ActionName.RUN_FULL_QUERY_AGAIN:
63
+ results = run_query_on_database() # EventReduce could not optimize this event
64
+ else:
65
+ run_action(action, query_params, change_event, results)
66
+ # -> [a, c, b]
67
+ ```
68
+
69
+ `dict_query_params()` is for dict documents, like the `primaryKey` and `sortFields` of the JavaScript implementation.
70
+ The sort fields can be dot-separated paths like `"address.city"`.
71
+ For other documents, create the `QueryParams` with a `get_document_id` function
72
+ and a `sort_fields_changed(previous, doc)` function that returns true if a sort field changed.
73
+
74
+ ### Key document map
75
+
76
+ For big result sets, pass a dict from the id to the document as `key_document_map`
77
+ to both `StateResolveFunctionInput` and `run_action()`.
78
+ It is used to check whether a document is in the results without scanning them,
79
+ and `run_action()` keeps it in sync with the results.
80
+
81
+ ### Differences to the JavaScript implementation
82
+
83
+ - Change events are created with `ChangeEvent.insert()`, `ChangeEvent.update()` and `ChangeEvent.delete()`.
84
+ - The names are snake_case, `ActionName` and `StateName` are enums whose values are the JavaScript names.
85
+ - `run_action()` raises an `EventReduceError` for `RUN_FULL_QUERY_AGAIN` and `UNKNOWN_ACTION`.
86
+ `ALWAYS_WRONG` is only used by the truth-table generator and raises like `UNKNOWN_ACTION`.
87
+
88
+ ## Result window buffer
89
+
90
+ For a query with a limit or a skip, EventReduce returns `RUN_FULL_QUERY_AGAIN` when a row has to move into the result window that is not in the previous results:
91
+ the row after the window (when a row leaves a full window) or the last skipped row (when a row is added in front of the window).
92
+ The result window buffer avoids most of these full queries: it keeps `limit_buffer_size` rows after the window and `skip_buffer_size` skipped rows before it in memory.
93
+ It works like the [result window buffer of the JavaScript implementation](../docs/result-window-buffer.md),
94
+ which explains how it works, how to choose the buffer sizes and has measurements.
95
+
96
+ ```python
97
+ from event_reduce import ResultWindowBuffer, get_recommended_result_window_buffer_size
98
+
99
+ # recommended size: about log2(limit), at most 30
100
+ size = get_recommended_result_window_buffer_size(query_params.limit)
101
+ buffer = ResultWindowBuffer(query_params, limit_buffer_size=size, skip_buffer_size=size)
102
+ # run the query with the skip and limit of the buffer
103
+ buffer.reset(run_query(buffer.query()))
104
+ # the rows of the original skip and limit
105
+ results = buffer.results()
106
+
107
+ # on new change events
108
+ result = buffer.apply_change_events(change_events)
109
+ if result.run_full_query_again:
110
+ buffer.reset(run_query(buffer.query()))
111
+ results = buffer.results()
112
+ elif result.changed:
113
+ results = result.results
114
+ ```
115
+
116
+ `query()` returns the `skip` and `limit` to run the query with, `limit` is None if all rows after the skip are needed.
117
+ `math.inf` as a buffer size keeps all rows on that side, like `Infinity` in JavaScript.
118
+
119
+ ## Development
120
+
121
+ ```sh
122
+ pip install pytest ruff mypy
123
+ pytest
124
+ ruff check . && ruff format --check . && mypy src
125
+ ```
126
+
127
+ The BDD is generated by the truth-table generator of the JavaScript implementation,
128
+ which also writes `src/event_reduce/bdd_generated.py`.
129
+ The tests run the [shared test suite](../test-suite/), the same tests as all other implementations.
130
+
131
+ ## Who uses EventReduce
132
+
133
+ [RxDB](https://rxdb.info/), the local-first database, uses EventReduce for its live queries since [version 9.0.0](https://rxdb.info/releases/9.0.0.html), where it is enabled by default.
134
+ When an app subscribes to a query, RxDB keeps the results up to date by running each write event through EventReduce,
135
+ and only queries the storage again when EventReduce returns `RUN_FULL_QUERY_AGAIN`.
@@ -0,0 +1,13 @@
1
+ event_reduce/__init__.py,sha256=dPlXb39ukXvg53yXaKX1Cdms6UBmqyLUWbzleGo0J2A,2530
2
+ event_reduce/actions.py,sha256=miJl5LPsJ5l9CMNpUa_DYn_NWSwdqQBI6n-ZdMc7pMo,6352
3
+ event_reduce/bdd.py,sha256=KYq1BAOMbFxK5kEHNkm4yUqv7I_1YdROFyZHcRJZNcs,1356
4
+ event_reduce/bdd_generated.py,sha256=sWbodc7xt9OuoTg-qv_szzxc4P6Pn_wskkMnw7byHH8,1280
5
+ event_reduce/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ event_reduce/result_window_buffer.py,sha256=wMdjjQijGAeq6BRweqIG-MTbhVNm6G_29ofIoFM0qyQ,15964
7
+ event_reduce/states.py,sha256=gGoYZexiVXH3G_svWAzvTqT2YHjC5xz-cxIxmhbdHhs,6187
8
+ event_reduce/types.py,sha256=wonFGpccRXtR5tgr8WBZBnpAlKlWW31p774g1HdjFHI,4784
9
+ event_reduce/util.py,sha256=WIqR9g168bWwZMrapgkgDpFqqClSbE5V8_ROVLLFO6E,3569
10
+ event_reduce-7.4.2.dist-info/METADATA,sha256=MPe8oJW_J6IBSK_UvbxX8ZEkANHc5p-k7M5V5Phms_E,5548
11
+ event_reduce-7.4.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
12
+ event_reduce-7.4.2.dist-info/top_level.txt,sha256=zSKcgKrau6mGJVMEFS-H0wBrPsa_7-S3W3t1Y_aircA,13
13
+ event_reduce-7.4.2.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ event_reduce