koi-net 1.0.0b8__py3-none-any.whl → 1.0.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- koi_net/core.py +5 -5
- koi_net/network/graph.py +4 -4
- koi_net/network/interface.py +17 -17
- koi_net/network/request_handler.py +27 -19
- koi_net/processor/default_handlers.py +22 -29
- koi_net/processor/handler.py +9 -2
- koi_net/processor/interface.py +58 -33
- koi_net/processor/knowledge_object.py +1 -1
- koi_net/protocol/event.py +3 -0
- {koi_net-1.0.0b8.dist-info → koi_net-1.0.0b10.dist-info}/METADATA +183 -10
- koi_net-1.0.0b10.dist-info/RECORD +24 -0
- koi_net-1.0.0b8.dist-info/RECORD +0 -24
- {koi_net-1.0.0b8.dist-info → koi_net-1.0.0b10.dist-info}/WHEEL +0 -0
- {koi_net-1.0.0b8.dist-info → koi_net-1.0.0b10.dist-info}/licenses/LICENSE +0 -0
koi_net/core.py
CHANGED
|
@@ -84,15 +84,15 @@ class NodeInterface:
|
|
|
84
84
|
)
|
|
85
85
|
)
|
|
86
86
|
|
|
87
|
-
logger.
|
|
87
|
+
logger.debug("Waiting for kobj queue to empty")
|
|
88
88
|
if self.use_kobj_processor_thread:
|
|
89
89
|
self.processor.kobj_queue.join()
|
|
90
90
|
else:
|
|
91
91
|
self.processor.flush_kobj_queue()
|
|
92
|
-
logger.
|
|
92
|
+
logger.debug("Done")
|
|
93
93
|
|
|
94
94
|
if not self.network.graph.get_neighbors() and self.first_contact:
|
|
95
|
-
logger.
|
|
95
|
+
logger.debug(f"I don't have any neighbors, reaching out to first contact {self.first_contact}")
|
|
96
96
|
|
|
97
97
|
events = [
|
|
98
98
|
Event.from_rid(EventType.FORGET, self.identity.rid),
|
|
@@ -106,7 +106,7 @@ class NodeInterface:
|
|
|
106
106
|
)
|
|
107
107
|
|
|
108
108
|
except httpx.ConnectError:
|
|
109
|
-
logger.
|
|
109
|
+
logger.warning("Failed to reach first contact")
|
|
110
110
|
return
|
|
111
111
|
|
|
112
112
|
|
|
@@ -118,7 +118,7 @@ class NodeInterface:
|
|
|
118
118
|
logger.info("Stopping node...")
|
|
119
119
|
|
|
120
120
|
if self.use_kobj_processor_thread:
|
|
121
|
-
logger.info("Waiting for kobj queue to empty")
|
|
121
|
+
logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
|
|
122
122
|
self.processor.kobj_queue.join()
|
|
123
123
|
else:
|
|
124
124
|
self.processor.flush_kobj_queue()
|
koi_net/network/graph.py
CHANGED
|
@@ -25,12 +25,12 @@ class NetworkGraph:
|
|
|
25
25
|
|
|
26
26
|
def generate(self):
|
|
27
27
|
"""Generates directed graph from cached KOI nodes and edges."""
|
|
28
|
-
logger.
|
|
28
|
+
logger.debug("Generating network graph")
|
|
29
29
|
self.dg.clear()
|
|
30
30
|
for rid in self.cache.list_rids():
|
|
31
31
|
if type(rid) == KoiNetNode:
|
|
32
32
|
self.dg.add_node(rid)
|
|
33
|
-
logger.
|
|
33
|
+
logger.debug(f"Added node {rid}")
|
|
34
34
|
|
|
35
35
|
elif type(rid) == KoiNetEdge:
|
|
36
36
|
edge_profile = self.get_edge_profile(rid)
|
|
@@ -38,8 +38,8 @@ class NetworkGraph:
|
|
|
38
38
|
logger.warning(f"Failed to load {rid!r}")
|
|
39
39
|
continue
|
|
40
40
|
self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
|
|
41
|
-
logger.
|
|
42
|
-
logger.
|
|
41
|
+
logger.debug(f"Added edge {rid} ({edge_profile.source} -> {edge_profile.target})")
|
|
42
|
+
logger.debug("Done")
|
|
43
43
|
|
|
44
44
|
def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
|
|
45
45
|
"""Returns node profile given its RID."""
|
koi_net/network/interface.py
CHANGED
|
@@ -100,7 +100,7 @@ class NetworkInterface:
|
|
|
100
100
|
|
|
101
101
|
Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
|
|
102
102
|
"""
|
|
103
|
-
logger.
|
|
103
|
+
logger.debug(f"Pushing event {event.event_type} {event.rid} to {node}")
|
|
104
104
|
|
|
105
105
|
node_profile = self.graph.get_node_profile(node)
|
|
106
106
|
if not node_profile:
|
|
@@ -136,14 +136,14 @@ class NetworkInterface:
|
|
|
136
136
|
if queue:
|
|
137
137
|
while not queue.empty():
|
|
138
138
|
event = queue.get()
|
|
139
|
-
logger.
|
|
139
|
+
logger.debug(f"Dequeued {event.event_type} '{event.rid}'")
|
|
140
140
|
events.append(event)
|
|
141
141
|
|
|
142
142
|
return events
|
|
143
143
|
|
|
144
144
|
def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
|
|
145
145
|
"""Flushes a node's poll queue, returning list of events."""
|
|
146
|
-
logger.
|
|
146
|
+
logger.debug(f"Flushing poll queue for {node}")
|
|
147
147
|
return self._flush_queue(self.poll_event_queue, node)
|
|
148
148
|
|
|
149
149
|
def flush_webhook_queue(self, node: KoiNetNode):
|
|
@@ -152,7 +152,7 @@ class NetworkInterface:
|
|
|
152
152
|
If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
|
|
153
153
|
"""
|
|
154
154
|
|
|
155
|
-
logger.
|
|
155
|
+
logger.debug(f"Flushing webhook queue for {node}")
|
|
156
156
|
|
|
157
157
|
node_profile = self.graph.get_node_profile(node)
|
|
158
158
|
|
|
@@ -167,7 +167,7 @@ class NetworkInterface:
|
|
|
167
167
|
events = self._flush_queue(self.webhook_event_queue, node)
|
|
168
168
|
if not events: return
|
|
169
169
|
|
|
170
|
-
logger.
|
|
170
|
+
logger.debug(f"Broadcasting {len(events)} events")
|
|
171
171
|
|
|
172
172
|
try:
|
|
173
173
|
self.request_handler.broadcast_events(node, events=events)
|
|
@@ -179,23 +179,23 @@ class NetworkInterface:
|
|
|
179
179
|
def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
|
|
180
180
|
"""Returns list of node RIDs which provide state for the specified RID type."""
|
|
181
181
|
|
|
182
|
-
logger.
|
|
182
|
+
logger.debug(f"Looking for state providers of '{rid_type}'")
|
|
183
183
|
provider_nodes = []
|
|
184
184
|
for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
|
|
185
185
|
node = self.graph.get_node_profile(node_rid)
|
|
186
186
|
|
|
187
187
|
if node.node_type == NodeType.FULL and rid_type in node.provides.state:
|
|
188
|
-
logger.
|
|
188
|
+
logger.debug(f"Found provider '{node_rid}'")
|
|
189
189
|
provider_nodes.append(node_rid)
|
|
190
190
|
|
|
191
191
|
if not provider_nodes:
|
|
192
|
-
logger.
|
|
192
|
+
logger.debug("Failed to find providers")
|
|
193
193
|
return provider_nodes
|
|
194
194
|
|
|
195
195
|
def fetch_remote_bundle(self, rid: RID):
|
|
196
196
|
"""Attempts to fetch a bundle by RID from known peer nodes."""
|
|
197
197
|
|
|
198
|
-
logger.
|
|
198
|
+
logger.debug(f"Fetching remote bundle '{rid}'")
|
|
199
199
|
remote_bundle = None
|
|
200
200
|
for node_rid in self.get_state_providers(type(rid)):
|
|
201
201
|
payload = self.request_handler.fetch_bundles(
|
|
@@ -203,7 +203,7 @@ class NetworkInterface:
|
|
|
203
203
|
|
|
204
204
|
if payload.bundles:
|
|
205
205
|
remote_bundle = payload.bundles[0]
|
|
206
|
-
logger.
|
|
206
|
+
logger.debug(f"Got bundle from '{node_rid}'")
|
|
207
207
|
break
|
|
208
208
|
|
|
209
209
|
if not remote_bundle:
|
|
@@ -214,7 +214,7 @@ class NetworkInterface:
|
|
|
214
214
|
def fetch_remote_manifest(self, rid: RID):
|
|
215
215
|
"""Attempts to fetch a manifest by RID from known peer nodes."""
|
|
216
216
|
|
|
217
|
-
logger.
|
|
217
|
+
logger.debug(f"Fetching remote manifest '{rid}'")
|
|
218
218
|
remote_manifest = None
|
|
219
219
|
for node_rid in self.get_state_providers(type(rid)):
|
|
220
220
|
payload = self.request_handler.fetch_manifests(
|
|
@@ -222,7 +222,7 @@ class NetworkInterface:
|
|
|
222
222
|
|
|
223
223
|
if payload.manifests:
|
|
224
224
|
remote_manifest = payload.manifests[0]
|
|
225
|
-
logger.
|
|
225
|
+
logger.debug(f"Got bundle from '{node_rid}'")
|
|
226
226
|
break
|
|
227
227
|
|
|
228
228
|
if not remote_manifest:
|
|
@@ -239,17 +239,17 @@ class NetworkInterface:
|
|
|
239
239
|
neighbors = self.graph.get_neighbors()
|
|
240
240
|
|
|
241
241
|
if not neighbors and self.first_contact:
|
|
242
|
-
logger.
|
|
242
|
+
logger.debug("No neighbors found, polling first contact")
|
|
243
243
|
try:
|
|
244
244
|
payload = self.request_handler.poll_events(
|
|
245
245
|
url=self.first_contact,
|
|
246
246
|
rid=self.identity.rid
|
|
247
247
|
)
|
|
248
248
|
if payload.events:
|
|
249
|
-
logger.
|
|
249
|
+
logger.debug(f"Received {len(payload.events)} events from '{self.first_contact}'")
|
|
250
250
|
return payload.events
|
|
251
251
|
except httpx.ConnectError:
|
|
252
|
-
logger.
|
|
252
|
+
logger.debug(f"Failed to reach first contact '{self.first_contact}'")
|
|
253
253
|
|
|
254
254
|
events = []
|
|
255
255
|
for node_rid in neighbors:
|
|
@@ -263,10 +263,10 @@ class NetworkInterface:
|
|
|
263
263
|
rid=self.identity.rid
|
|
264
264
|
)
|
|
265
265
|
if payload.events:
|
|
266
|
-
logger.
|
|
266
|
+
logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
|
|
267
267
|
events.extend(payload.events)
|
|
268
268
|
except httpx.ConnectError:
|
|
269
|
-
logger.
|
|
269
|
+
logger.debug(f"Failed to reach node '{node_rid}'")
|
|
270
270
|
continue
|
|
271
271
|
|
|
272
272
|
return events
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import httpx
|
|
3
|
-
from pydantic import BaseModel
|
|
4
3
|
from rid_lib import RID
|
|
5
4
|
from rid_lib.ext import Cache
|
|
6
5
|
from rid_lib.types.koi_net_node import KoiNetNode
|
|
@@ -46,7 +45,7 @@ class RequestHandler:
|
|
|
46
45
|
request: RequestModels,
|
|
47
46
|
response_model: type[ResponseModels] | None = None
|
|
48
47
|
) -> ResponseModels | None:
|
|
49
|
-
logger.
|
|
48
|
+
logger.debug(f"Making request to {url}")
|
|
50
49
|
resp = httpx.post(
|
|
51
50
|
url=url,
|
|
52
51
|
data=request.model_dump_json()
|
|
@@ -66,7 +65,7 @@ class RequestHandler:
|
|
|
66
65
|
raise Exception("Node not found")
|
|
67
66
|
if node_profile.node_type != NodeType.FULL:
|
|
68
67
|
raise Exception("Can't query partial node")
|
|
69
|
-
logger.
|
|
68
|
+
logger.debug(f"Resolved {node_rid!r} to {node_profile.base_url}")
|
|
70
69
|
return node_profile.base_url
|
|
71
70
|
else:
|
|
72
71
|
return url
|
|
@@ -79,10 +78,11 @@ class RequestHandler:
|
|
|
79
78
|
**kwargs
|
|
80
79
|
) -> None:
|
|
81
80
|
"""See protocol.api_models.EventsPayload for available kwargs."""
|
|
81
|
+
request = req or EventsPayload.model_validate(kwargs)
|
|
82
82
|
self.make_request(
|
|
83
|
-
self.get_url(node, url) + BROADCAST_EVENTS_PATH,
|
|
84
|
-
req or EventsPayload.model_validate(kwargs)
|
|
83
|
+
self.get_url(node, url) + BROADCAST_EVENTS_PATH, request
|
|
85
84
|
)
|
|
85
|
+
logger.info(f"Broadcasted {len(request.events)} event(s) to {node or url!r}")
|
|
86
86
|
|
|
87
87
|
def poll_events(
|
|
88
88
|
self,
|
|
@@ -92,12 +92,14 @@ class RequestHandler:
|
|
|
92
92
|
**kwargs
|
|
93
93
|
) -> EventsPayload:
|
|
94
94
|
"""See protocol.api_models.PollEvents for available kwargs."""
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
request = req or PollEvents.model_validate(kwargs)
|
|
96
|
+
resp = self.make_request(
|
|
97
|
+
self.get_url(node, url) + POLL_EVENTS_PATH, request,
|
|
98
98
|
response_model=EventsPayload
|
|
99
99
|
)
|
|
100
|
-
|
|
100
|
+
logger.info(f"Polled {len(resp.events)} events from {node or url!r}")
|
|
101
|
+
return resp
|
|
102
|
+
|
|
101
103
|
def fetch_rids(
|
|
102
104
|
self,
|
|
103
105
|
node: RID = None,
|
|
@@ -106,11 +108,13 @@ class RequestHandler:
|
|
|
106
108
|
**kwargs
|
|
107
109
|
) -> RidsPayload:
|
|
108
110
|
"""See protocol.api_models.FetchRids for available kwargs."""
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
111
|
+
request = req or FetchRids.model_validate(kwargs)
|
|
112
|
+
resp = self.make_request(
|
|
113
|
+
self.get_url(node, url) + FETCH_RIDS_PATH, request,
|
|
112
114
|
response_model=RidsPayload
|
|
113
115
|
)
|
|
116
|
+
logger.info(f"Fetched {len(resp.rids)} RID(s) from {node or url!r}")
|
|
117
|
+
return resp
|
|
114
118
|
|
|
115
119
|
def fetch_manifests(
|
|
116
120
|
self,
|
|
@@ -120,11 +124,13 @@ class RequestHandler:
|
|
|
120
124
|
**kwargs
|
|
121
125
|
) -> ManifestsPayload:
|
|
122
126
|
"""See protocol.api_models.FetchManifests for available kwargs."""
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
127
|
+
request = req or FetchManifests.model_validate(kwargs)
|
|
128
|
+
resp = self.make_request(
|
|
129
|
+
self.get_url(node, url) + FETCH_MANIFESTS_PATH, request,
|
|
126
130
|
response_model=ManifestsPayload
|
|
127
131
|
)
|
|
132
|
+
logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node or url!r}")
|
|
133
|
+
return resp
|
|
128
134
|
|
|
129
135
|
def fetch_bundles(
|
|
130
136
|
self,
|
|
@@ -134,8 +140,10 @@ class RequestHandler:
|
|
|
134
140
|
**kwargs
|
|
135
141
|
) -> BundlesPayload:
|
|
136
142
|
"""See protocol.api_models.FetchBundles for available kwargs."""
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
143
|
+
request = req or FetchBundles.model_validate(kwargs)
|
|
144
|
+
resp = self.make_request(
|
|
145
|
+
self.get_url(node, url) + FETCH_BUNDLES_PATH, request,
|
|
140
146
|
response_model=BundlesPayload
|
|
141
|
-
)
|
|
147
|
+
)
|
|
148
|
+
logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node or url!r}")
|
|
149
|
+
return resp
|
|
@@ -22,24 +22,17 @@ def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
22
22
|
"""
|
|
23
23
|
if (kobj.rid == processor.identity.rid and
|
|
24
24
|
kobj.source == KnowledgeSource.External):
|
|
25
|
-
logger.
|
|
25
|
+
logger.debug("Don't let anyone else tell me who I am!")
|
|
26
26
|
return STOP_CHAIN
|
|
27
27
|
|
|
28
28
|
if kobj.event_type == EventType.FORGET:
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
kobj.normalized_event_type = EventType.FORGET
|
|
32
|
-
return kobj
|
|
33
|
-
|
|
34
|
-
else:
|
|
35
|
-
# can't forget something I don't know about
|
|
36
|
-
return STOP_CHAIN
|
|
37
|
-
|
|
29
|
+
kobj.normalized_event_type = EventType.FORGET
|
|
30
|
+
return kobj
|
|
38
31
|
|
|
39
32
|
# Manifest handlers
|
|
40
33
|
|
|
41
34
|
@KnowledgeHandler.create(HandlerType.Manifest)
|
|
42
|
-
def
|
|
35
|
+
def basic_manifest_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
43
36
|
"""Default manifest handler.
|
|
44
37
|
|
|
45
38
|
Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
|
|
@@ -48,17 +41,17 @@ def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
48
41
|
|
|
49
42
|
if prev_bundle:
|
|
50
43
|
if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
|
|
51
|
-
logger.
|
|
44
|
+
logger.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
|
|
52
45
|
return STOP_CHAIN
|
|
53
46
|
if kobj.manifest.timestamp <= prev_bundle.manifest.timestamp:
|
|
54
|
-
logger.
|
|
47
|
+
logger.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
|
|
55
48
|
return STOP_CHAIN
|
|
56
49
|
|
|
57
|
-
logger.
|
|
50
|
+
logger.debug("RID previously known to me, labeling as 'UPDATE'")
|
|
58
51
|
kobj.normalized_event_type = EventType.UPDATE
|
|
59
52
|
|
|
60
53
|
else:
|
|
61
|
-
logger.
|
|
54
|
+
logger.debug("RID previously unknown to me, labeling as 'NEW'")
|
|
62
55
|
kobj.normalized_event_type = EventType.NEW
|
|
63
56
|
|
|
64
57
|
return kobj
|
|
@@ -66,7 +59,11 @@ def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
66
59
|
|
|
67
60
|
# Bundle handlers
|
|
68
61
|
|
|
69
|
-
@KnowledgeHandler.create(
|
|
62
|
+
@KnowledgeHandler.create(
|
|
63
|
+
handler_type=HandlerType.Bundle,
|
|
64
|
+
rid_types=[KoiNetEdge],
|
|
65
|
+
source=KnowledgeSource.External,
|
|
66
|
+
event_types=[EventType.NEW, EventType.UPDATE])
|
|
70
67
|
def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
71
68
|
"""Handles basic edge negotiation process.
|
|
72
69
|
|
|
@@ -74,17 +71,13 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
74
71
|
"""
|
|
75
72
|
|
|
76
73
|
edge_profile = EdgeProfile.model_validate(kobj.contents)
|
|
77
|
-
|
|
78
|
-
# only want to handle external knowledge events (not edges this node created)
|
|
79
|
-
if kobj.source != KnowledgeSource.External:
|
|
80
|
-
return
|
|
81
74
|
|
|
82
75
|
# indicates peer subscribing to me
|
|
83
76
|
if edge_profile.source == processor.identity.rid:
|
|
84
77
|
if edge_profile.status != EdgeStatus.PROPOSED:
|
|
85
78
|
return
|
|
86
79
|
|
|
87
|
-
logger.
|
|
80
|
+
logger.debug("Handling edge negotiation")
|
|
88
81
|
|
|
89
82
|
peer_rid = edge_profile.target
|
|
90
83
|
peer_profile = processor.network.graph.get_node_profile(peer_rid)
|
|
@@ -103,11 +96,11 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
103
96
|
abort = False
|
|
104
97
|
if (edge_profile.edge_type == EdgeType.WEBHOOK and
|
|
105
98
|
peer_profile.node_type == NodeType.PARTIAL):
|
|
106
|
-
logger.
|
|
99
|
+
logger.debug("Partial nodes cannot use webhooks")
|
|
107
100
|
abort = True
|
|
108
101
|
|
|
109
102
|
if not set(edge_profile.rid_types).issubset(provided_events):
|
|
110
|
-
logger.
|
|
103
|
+
logger.debug("Requested RID types not provided by this node")
|
|
111
104
|
abort = True
|
|
112
105
|
|
|
113
106
|
if abort:
|
|
@@ -117,16 +110,16 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
117
110
|
|
|
118
111
|
else:
|
|
119
112
|
# approve edge profile
|
|
120
|
-
logger.
|
|
113
|
+
logger.debug("Approving proposed edge")
|
|
121
114
|
edge_profile.status = EdgeStatus.APPROVED
|
|
122
115
|
updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
|
|
123
116
|
|
|
124
|
-
processor.handle(bundle=updated_bundle)
|
|
117
|
+
processor.handle(bundle=updated_bundle, event_type=EventType.UPDATE)
|
|
125
118
|
return
|
|
126
119
|
|
|
127
120
|
elif edge_profile.target == processor.identity.rid:
|
|
128
121
|
if edge_profile.status == EdgeStatus.APPROVED:
|
|
129
|
-
logger.
|
|
122
|
+
logger.debug("Edge approved by other node!")
|
|
130
123
|
|
|
131
124
|
|
|
132
125
|
# Network handlers
|
|
@@ -147,12 +140,12 @@ def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeOb
|
|
|
147
140
|
edge_profile = kobj.bundle.validate_contents(EdgeProfile)
|
|
148
141
|
|
|
149
142
|
if edge_profile.source == processor.identity.rid:
|
|
150
|
-
logger.
|
|
143
|
+
logger.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
|
|
151
144
|
kobj.network_targets.update([edge_profile.target])
|
|
152
145
|
involves_me = True
|
|
153
146
|
|
|
154
147
|
elif edge_profile.target == processor.identity.rid:
|
|
155
|
-
logger.
|
|
148
|
+
logger.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
|
|
156
149
|
kobj.network_targets.update([edge_profile.source])
|
|
157
150
|
involves_me = True
|
|
158
151
|
|
|
@@ -163,7 +156,7 @@ def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeOb
|
|
|
163
156
|
allowed_type=type(kobj.rid)
|
|
164
157
|
)
|
|
165
158
|
|
|
166
|
-
logger.
|
|
159
|
+
logger.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
|
|
167
160
|
kobj.network_targets.update(subscribers)
|
|
168
161
|
|
|
169
162
|
return kobj
|
koi_net/processor/handler.py
CHANGED
|
@@ -3,6 +3,9 @@ from enum import StrEnum
|
|
|
3
3
|
from typing import Callable
|
|
4
4
|
from rid_lib import RIDType
|
|
5
5
|
|
|
6
|
+
from ..protocol.event import EventType
|
|
7
|
+
from .knowledge_object import KnowledgeSource, KnowledgeEventType
|
|
8
|
+
|
|
6
9
|
|
|
7
10
|
class StopChain:
|
|
8
11
|
"""Class for a sentinel value by knowledge handlers."""
|
|
@@ -34,19 +37,23 @@ class KnowledgeHandler:
|
|
|
34
37
|
func: Callable
|
|
35
38
|
handler_type: HandlerType
|
|
36
39
|
rid_types: list[RIDType] | None
|
|
40
|
+
source: KnowledgeSource | None = None
|
|
41
|
+
event_types: list[KnowledgeEventType] | None = None
|
|
37
42
|
|
|
38
43
|
@classmethod
|
|
39
44
|
def create(
|
|
40
45
|
cls,
|
|
41
46
|
handler_type: HandlerType,
|
|
42
|
-
rid_types: list[RIDType] | None = None
|
|
47
|
+
rid_types: list[RIDType] | None = None,
|
|
48
|
+
source: KnowledgeSource | None = None,
|
|
49
|
+
event_types: list[KnowledgeEventType] | None = None
|
|
43
50
|
):
|
|
44
51
|
"""Special decorator that returns a KnowledgeHandler instead of a function.
|
|
45
52
|
|
|
46
53
|
The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
|
|
47
54
|
"""
|
|
48
55
|
def decorator(func: Callable) -> KnowledgeHandler:
|
|
49
|
-
handler = cls(func, handler_type, rid_types)
|
|
56
|
+
handler = cls(func, handler_type, rid_types, source, event_types)
|
|
50
57
|
return handler
|
|
51
58
|
return decorator
|
|
52
59
|
|
koi_net/processor/interface.py
CHANGED
|
@@ -62,11 +62,13 @@ class ProcessorInterface:
|
|
|
62
62
|
def register_handler(
|
|
63
63
|
self,
|
|
64
64
|
handler_type: HandlerType,
|
|
65
|
-
rid_types: list[RIDType] | None = None
|
|
65
|
+
rid_types: list[RIDType] | None = None,
|
|
66
|
+
source: KnowledgeSource | None = None,
|
|
67
|
+
event_types: list[KnowledgeEventType] | None = None
|
|
66
68
|
):
|
|
67
69
|
"""Assigns decorated function as handler for this processor."""
|
|
68
70
|
def decorator(func: Callable) -> Callable:
|
|
69
|
-
handler = KnowledgeHandler(func, handler_type, rid_types)
|
|
71
|
+
handler = KnowledgeHandler(func, handler_type, rid_types, source, event_types)
|
|
70
72
|
self.add_handler(handler)
|
|
71
73
|
return func
|
|
72
74
|
return decorator
|
|
@@ -93,12 +95,18 @@ class ProcessorInterface:
|
|
|
93
95
|
if handler.rid_types and type(kobj.rid) not in handler.rid_types:
|
|
94
96
|
continue
|
|
95
97
|
|
|
96
|
-
|
|
98
|
+
if handler.source and handler.source != kobj.source:
|
|
99
|
+
continue
|
|
100
|
+
|
|
101
|
+
if handler.event_types and kobj.event_type not in handler.event_types:
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
|
|
97
105
|
resp = handler.func(self, kobj.model_copy())
|
|
98
106
|
|
|
99
107
|
# stops handler chain execution
|
|
100
108
|
if resp is STOP_CHAIN:
|
|
101
|
-
logger.
|
|
109
|
+
logger.debug(f"Handler chain stopped by {handler.func.__name__}")
|
|
102
110
|
return STOP_CHAIN
|
|
103
111
|
# kobj unmodified
|
|
104
112
|
elif resp is None:
|
|
@@ -106,7 +114,7 @@ class ProcessorInterface:
|
|
|
106
114
|
# kobj modified by handler
|
|
107
115
|
elif isinstance(resp, KnowledgeObject):
|
|
108
116
|
kobj = resp
|
|
109
|
-
logger.
|
|
117
|
+
logger.debug(f"Knowledge object modified by {handler.func.__name__}")
|
|
110
118
|
else:
|
|
111
119
|
raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
|
|
112
120
|
|
|
@@ -126,34 +134,41 @@ class ProcessorInterface:
|
|
|
126
134
|
The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
|
|
127
135
|
"""
|
|
128
136
|
|
|
129
|
-
logger.
|
|
137
|
+
logger.debug(f"Handling {kobj!r}")
|
|
130
138
|
kobj = self.call_handler_chain(HandlerType.RID, kobj)
|
|
131
139
|
if kobj is STOP_CHAIN: return
|
|
132
140
|
|
|
133
141
|
if kobj.event_type == EventType.FORGET:
|
|
134
142
|
bundle = self.cache.read(kobj.rid)
|
|
135
143
|
if not bundle:
|
|
136
|
-
logger.
|
|
144
|
+
logger.debug("Local bundle not found")
|
|
137
145
|
return
|
|
138
146
|
|
|
139
147
|
# the bundle (to be deleted) attached to kobj for downstream analysis
|
|
140
|
-
logger.
|
|
148
|
+
logger.debug("Adding local bundle (to be deleted) to knowledge object")
|
|
141
149
|
kobj.manifest = bundle.manifest
|
|
142
150
|
kobj.contents = bundle.contents
|
|
143
151
|
|
|
144
152
|
else:
|
|
145
153
|
# attempt to retrieve manifest
|
|
146
154
|
if not kobj.manifest:
|
|
155
|
+
logger.debug("Manifest not found")
|
|
147
156
|
if kobj.source == KnowledgeSource.External:
|
|
148
|
-
logger.
|
|
157
|
+
logger.debug("Attempting to fetch remote manifest")
|
|
149
158
|
manifest = self.network.fetch_remote_manifest(kobj.rid)
|
|
150
|
-
if not manifest: return
|
|
151
159
|
|
|
152
160
|
elif kobj.source == KnowledgeSource.Internal:
|
|
153
|
-
logger.
|
|
161
|
+
logger.debug("Attempting to read manifest from cache")
|
|
154
162
|
bundle = self.cache.read(kobj.rid)
|
|
155
|
-
if
|
|
156
|
-
|
|
163
|
+
if bundle:
|
|
164
|
+
manifest = bundle.manifest
|
|
165
|
+
else:
|
|
166
|
+
manifest = None
|
|
167
|
+
return
|
|
168
|
+
|
|
169
|
+
if not manifest:
|
|
170
|
+
logger.debug("Failed to find manifest")
|
|
171
|
+
return
|
|
157
172
|
|
|
158
173
|
kobj.manifest = manifest
|
|
159
174
|
|
|
@@ -162,48 +177,51 @@ class ProcessorInterface:
|
|
|
162
177
|
|
|
163
178
|
# attempt to retrieve bundle
|
|
164
179
|
if not kobj.bundle:
|
|
180
|
+
logger.debug("Bundle not found")
|
|
165
181
|
if kobj.source == KnowledgeSource.External:
|
|
166
|
-
logger.
|
|
182
|
+
logger.debug("Attempting to fetch remote bundle")
|
|
167
183
|
bundle = self.network.fetch_remote_bundle(kobj.rid)
|
|
168
|
-
# TODO: WARNING MANIFEST MAY BE DIFFERENT
|
|
169
184
|
|
|
170
185
|
elif kobj.source == KnowledgeSource.Internal:
|
|
171
|
-
logger.
|
|
186
|
+
logger.debug("Attempting to read bundle from cache")
|
|
172
187
|
bundle = self.cache.read(kobj.rid)
|
|
173
188
|
|
|
174
189
|
if kobj.manifest != bundle.manifest:
|
|
175
190
|
logger.warning("Retrieved bundle contains a different manifest")
|
|
176
191
|
|
|
177
|
-
if not bundle:
|
|
192
|
+
if not bundle:
|
|
193
|
+
logger.debug("Failed to find bundle")
|
|
194
|
+
return
|
|
195
|
+
|
|
178
196
|
kobj.manifest = bundle.manifest
|
|
179
197
|
kobj.contents = bundle.contents
|
|
180
198
|
|
|
181
|
-
|
|
182
|
-
|
|
199
|
+
kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
|
|
200
|
+
if kobj is STOP_CHAIN: return
|
|
183
201
|
|
|
184
202
|
if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
|
|
185
|
-
logger.info(f"Writing {kobj!r}
|
|
203
|
+
logger.info(f"Writing to cache: {kobj!r}")
|
|
186
204
|
self.cache.write(kobj.bundle)
|
|
187
205
|
|
|
188
206
|
elif kobj.normalized_event_type == EventType.FORGET:
|
|
189
|
-
logger.info(f"Deleting {kobj!r}
|
|
207
|
+
logger.info(f"Deleting from cache: {kobj!r}")
|
|
190
208
|
self.cache.delete(kobj.rid)
|
|
191
209
|
|
|
192
210
|
else:
|
|
193
|
-
logger.
|
|
211
|
+
logger.debug("Normalized event type was never set, no cache or network operations will occur")
|
|
194
212
|
return
|
|
195
213
|
|
|
196
214
|
if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
|
|
197
|
-
logger.
|
|
215
|
+
logger.debug("Change to node or edge, regenerating network graph")
|
|
198
216
|
self.network.graph.generate()
|
|
199
217
|
|
|
200
218
|
kobj = self.call_handler_chain(HandlerType.Network, kobj)
|
|
201
219
|
if kobj is STOP_CHAIN: return
|
|
202
220
|
|
|
203
221
|
if kobj.network_targets:
|
|
204
|
-
logger.
|
|
222
|
+
logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
|
|
205
223
|
else:
|
|
206
|
-
logger.
|
|
224
|
+
logger.debug("No network targets set")
|
|
207
225
|
|
|
208
226
|
for node in kobj.network_targets:
|
|
209
227
|
self.network.push_event_to(kobj.normalized_event, node, flush=True)
|
|
@@ -220,18 +238,25 @@ class ProcessorInterface:
|
|
|
220
238
|
|
|
221
239
|
while not self.kobj_queue.empty():
|
|
222
240
|
kobj = self.kobj_queue.get()
|
|
223
|
-
logger.
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
241
|
+
logger.debug(f"Dequeued {kobj!r}")
|
|
242
|
+
|
|
243
|
+
try:
|
|
244
|
+
self.process_kobj(kobj)
|
|
245
|
+
finally:
|
|
246
|
+
self.kobj_queue.task_done()
|
|
247
|
+
logger.debug("Done")
|
|
227
248
|
|
|
228
249
|
def kobj_processor_worker(self, timeout=0.1):
|
|
229
250
|
while True:
|
|
230
251
|
try:
|
|
231
252
|
kobj = self.kobj_queue.get(timeout=timeout)
|
|
232
|
-
logger.
|
|
233
|
-
|
|
234
|
-
|
|
253
|
+
logger.debug(f"Dequeued {kobj!r}")
|
|
254
|
+
|
|
255
|
+
try:
|
|
256
|
+
self.process_kobj(kobj)
|
|
257
|
+
finally:
|
|
258
|
+
self.kobj_queue.task_done()
|
|
259
|
+
logger.debug("Done")
|
|
235
260
|
|
|
236
261
|
except queue.Empty:
|
|
237
262
|
pass
|
|
@@ -267,4 +292,4 @@ class ProcessorInterface:
|
|
|
267
292
|
raise ValueError("One of 'rid', 'manifest', 'bundle', 'event', or 'kobj' must be provided")
|
|
268
293
|
|
|
269
294
|
self.kobj_queue.put(_kobj)
|
|
270
|
-
logger.
|
|
295
|
+
logger.debug(f"Queued {_kobj!r}")
|
|
@@ -35,7 +35,7 @@ class KnowledgeObject(BaseModel):
|
|
|
35
35
|
network_targets: set[KoiNetNode] = set()
|
|
36
36
|
|
|
37
37
|
def __repr__(self):
|
|
38
|
-
return f"<
|
|
38
|
+
return f"<KObj '{self.rid}' event type: '{self.event_type}' -> '{self.normalized_event_type}', source: '{self.source}'>"
|
|
39
39
|
|
|
40
40
|
@classmethod
|
|
41
41
|
def from_rid(
|
koi_net/protocol/event.py
CHANGED
|
@@ -15,6 +15,9 @@ class Event(BaseModel):
|
|
|
15
15
|
manifest: Manifest | None = None
|
|
16
16
|
contents: dict | None = None
|
|
17
17
|
|
|
18
|
+
def __repr__(self):
|
|
19
|
+
return f"<Event '{self.rid}' event type: '{self.event_type}'>"
|
|
20
|
+
|
|
18
21
|
@classmethod
|
|
19
22
|
def from_bundle(cls, event_type: EventType, bundle: Bundle):
|
|
20
23
|
return cls(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: koi-net
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.0b10
|
|
4
4
|
Summary: Implementation of KOI-net protocol in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/BlockScience/koi-net/
|
|
6
6
|
Author-email: Luke Miller <luke@block.science>
|
|
@@ -44,6 +44,37 @@ Description-Content-Type: text/markdown
|
|
|
44
44
|
|
|
45
45
|
*This specification is the result of several iterations of KOI research, [read more here](https://github.com/BlockScience/koi).*
|
|
46
46
|
|
|
47
|
+
### Jump to Sections:
|
|
48
|
+
- [Protocol](#protocol)
|
|
49
|
+
- [Introduction](#introduction)
|
|
50
|
+
- [Communication Methods](#communication-methods)
|
|
51
|
+
- [Quickstart](#quickstart)
|
|
52
|
+
- [Setup](#setup)
|
|
53
|
+
- [Creating a Node](#creating-a-node)
|
|
54
|
+
- [Knowledge Processing](#knowledge-processing)
|
|
55
|
+
- [Try It Out!](#try-it-out)
|
|
56
|
+
- [Advanced](#advanced)
|
|
57
|
+
- [Knowledge Processing Pipeline](#knowledge-processing-pipeline)
|
|
58
|
+
- [Knowledge Handlers](#knowledge-handlers)
|
|
59
|
+
- [RID Handler](#rid-handler)
|
|
60
|
+
- [Manifest Handler](#manifest-handler)
|
|
61
|
+
- [Bundle Handler](#bundle-handler)
|
|
62
|
+
- [Network Handler](#network-handler)
|
|
63
|
+
- [Final Handler](#final-handler)
|
|
64
|
+
- [Registering Handlers](#registering-handlers)
|
|
65
|
+
- [Default Behavior](#default-behavior)
|
|
66
|
+
- [Implementation Reference](#implementation-reference)
|
|
67
|
+
- [Node Interface](#node-interface)
|
|
68
|
+
- [Node Identity](#node-identity)
|
|
69
|
+
- [Network Interface](#network-interface)
|
|
70
|
+
- [Network Graph](#network-graph)
|
|
71
|
+
- [Request Handler](#request-handler)
|
|
72
|
+
- [Response Handler](#response-handler)
|
|
73
|
+
- [Processor Interface](#processor-interface)
|
|
74
|
+
- [Development](#development)
|
|
75
|
+
- [Setup](#setup-1)
|
|
76
|
+
- [Distribution](#distribution)
|
|
77
|
+
|
|
47
78
|
# Protocol
|
|
48
79
|
## Introduction
|
|
49
80
|
|
|
@@ -87,10 +118,10 @@ The request and payload JSON objects are composed of the fundamental "knowledge
|
|
|
87
118
|
}
|
|
88
119
|
```
|
|
89
120
|
|
|
90
|
-
This means that events are essentially just an RID, manifest, or bundle with an event type attached. Event types can be one of `FORGET`, `UPDATE`, or `NEW` forming the "FUN" acronym. While these types roughly correspond to delete, update, and create from CRUD operations, but they are not commands, they are signals. A node emits an event to indicate that its internal state has changed:
|
|
91
|
-
- `NEW` - indicates an previously unknown RID was cached
|
|
92
|
-
- `UPDATE` - indicates a previously known RID was cached
|
|
93
|
-
- `FORGET` - indicates a previously known RID was deleted
|
|
121
|
+
This means that events are essentially just an RID, manifest, or bundle with an event type attached. Event types can be one of `"FORGET"`, `"UPDATE"`, or `"NEW"` forming the "FUN" acronym. While these types roughly correspond to delete, update, and create from CRUD operations, but they are not commands, they are signals. A node emits an event to indicate that its internal state has changed:
|
|
122
|
+
- `"NEW"` - indicates an previously unknown RID was cached
|
|
123
|
+
- `"UPDATE"` - indicates a previously known RID was cached
|
|
124
|
+
- `"FORGET"` - indicates a previously known RID was deleted
|
|
94
125
|
|
|
95
126
|
Nodes may broadcast events to other nodes to indicate their internal state changed. Conversely, nodes may also listen to events from other nodes and as a result decide to change their internal state, take some other action, or do nothing.
|
|
96
127
|
|
|
@@ -149,12 +180,14 @@ node = NodeInterface(
|
|
|
149
180
|
)
|
|
150
181
|
```
|
|
151
182
|
|
|
183
|
+
When creating a node, you optionally enable `use_kobj_processor_thread` which will run the knowledge processing pipeline on a separate thread. This thread will automatically dequeue and process knowledge objects as they are added to the `kobj_queue`, which happenes when you call `node.process.handle(...)`. This is required to prevent race conditions in asynchronous applications, like web servers, therefore it is recommended to enable this feature for all full nodes.
|
|
184
|
+
|
|
152
185
|
## Knowledge Processing
|
|
153
186
|
|
|
154
187
|
Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.start()` and `node.stop()` at the beginning and end of your node's life cycle.
|
|
155
188
|
|
|
156
189
|
### Partial Node
|
|
157
|
-
Make sure to set `source=KnowledgeSource.External
|
|
190
|
+
Make sure to set `source=KnowledgeSource.External` when calling `handle` on external knowledge, this indicates to the knowledge processing pipeline that the incoming knowledge was received from another node. Where the knowledge is sourced from will impact decisions in the node's knowledge handlers.
|
|
158
191
|
```python
|
|
159
192
|
import time
|
|
160
193
|
from koi_net.processor.knowledge_object import KnowledgeSource
|
|
@@ -252,6 +285,147 @@ python -m examples.basic_coordinator_node
|
|
|
252
285
|
python -m examples.basic_partial_node
|
|
253
286
|
```
|
|
254
287
|
|
|
288
|
+
# Advanced
|
|
289
|
+
|
|
290
|
+
## Knowledge Processing Pipeline
|
|
291
|
+
Beyond the `NodeInterface` setup and boiler plate for partial/full nodes, node behavior is mostly controlled through the use of knowledge handlers. Effectively creating your own handlers relies on a solid understanding of the knowledge processing pipeline, so we'll start with that. As a developer, you will interface with the pipeline through the `ProcessorInterface` accessed with `node.processor`. The pipeline handles knowledge objects, from the `KnowledgeObject` class, a container for all knowledge types in the RID / KOI-net ecosystem:
|
|
292
|
+
- RIDs
|
|
293
|
+
- Manifests
|
|
294
|
+
- Bundles
|
|
295
|
+
- Events
|
|
296
|
+
|
|
297
|
+
Here is the class definition for a knowledge object:
|
|
298
|
+
```python
|
|
299
|
+
type KnowledgeEventType = EventType | None
|
|
300
|
+
|
|
301
|
+
class KnowledgeSource(StrEnum):
|
|
302
|
+
Internal = "INTERNAL"
|
|
303
|
+
External = "EXTERNAL"
|
|
304
|
+
|
|
305
|
+
class KnowledgeObject(BaseModel):
|
|
306
|
+
rid: RID
|
|
307
|
+
manifest: Manifest | None = None
|
|
308
|
+
contents: dict | None = None
|
|
309
|
+
event_type: KnowledgeEventType = None
|
|
310
|
+
source: KnowledgeSource
|
|
311
|
+
normalized_event_type: KnowledgeEventType = None
|
|
312
|
+
network_targets: set[KoiNetNode] = set()
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
In addition to the fields required to represent the knowledge types (`rid`, `manifest`, `contents`, `event_type`), knowledge objects also include a `source` field, indicating whether the knowledge originated from within the node (`KnowledgeSource.Internal`) or from another node (`KnowledgeSource.External`).
|
|
316
|
+
|
|
317
|
+
The final two fields are not inputs, but are set by handlers as the knowledge object moves through the processing pipeline. The normalized event type indicates the event type normalized to the perspective of the node's cache, and the network targets indicate where the resulting event should be broadcasted to.
|
|
318
|
+
|
|
319
|
+
Knowledge objects enter the processing pipeline through the `node.processor.handle(...)` method. Using kwargs you can pass any of the knowledge types listed above, a knowledge source, and an optional `event_type` (for non-event knowledge types). The handle function will simply normalize the provided knowledge type into a knowledge object, and put it in the `kobj_queue`, an internal, thread-safe queue of knowledge objects. If you have enabled `use_kobj_processor_thread` then the queue will be automatically processed on the processor thread, otherwise you will need to regularly call `flush_kobj_queue` to process queued knowledge objects (as in the partial node example). Both methods will process knowledge objects sequentially, in the order that they were queued in (FIFO).
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
## Knowledge Handlers
|
|
323
|
+
|
|
324
|
+
Processing happens through five distinct phases, corresponding to the handler types: `RID`, `Manifest`, `Bundle`, `Network`, and `Final`. Each handler type can be understood by describing (1) what knowledge object fields are available to the handler, and (2) what action takes place after this phase, which the handler can influence. As knowledge objects pass through the pipeline, fields may be added or updated.
|
|
325
|
+
|
|
326
|
+
Handlers are registered in a single handler array within the processor. There is no limit to the number of handlers in use, and multiple handlers can be assigned to the same handler type. At each phase of knowledge processing, we will chain together all of the handlers of the corresponding type and run them in their array order. The order handlers are registered in matters!
|
|
327
|
+
|
|
328
|
+
Each handler will be passed a knowledge object. They can choose to return one of three types: `None`, `KnowledgeObject`, or `STOP_CHAIN`. Returning `None` will pass the unmodified knowledge object (the same one the handler received) to the next handler in the chain. If a handler modified their knowledge object, they should return it to pass the new version to the next handler. Finally, a handler can return `STOP_CHAIN` to immediately stop processing the knowledge object. No further handlers will be called and it will not enter the next phase of processing.
|
|
329
|
+
|
|
330
|
+
Summary of processing pipeline:
|
|
331
|
+
```
|
|
332
|
+
RID -> Manifest -> Bundle -> [cache action] -> Network -> [network action] -> Final
|
|
333
|
+
|
|
|
334
|
+
(skip if event type is "FORGET")
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
### RID Handler
|
|
338
|
+
The knowledge object passed to handlers of this type are guaranteed to have an RID and knowledge source field. This handler type acts as a filter, if none of the handlers return `STOP_CHAIN` the pipeline will progress to the next phase. The pipeline diverges slightly after this handler chain, based on the event type of the knowledge object.
|
|
339
|
+
|
|
340
|
+
If the event type is `"NEW"`, `"UPDATE"`, or `None` and the manifest is not already in the knowledge object, the node will attempt to retrieve it from (1) the local cache if the source is internal, or (2) from another node if the source is external. If it fails to retrieves the manifest, the pipeline will end. Next, the manifest handler chain will be called.
|
|
341
|
+
|
|
342
|
+
If the event type is `"FORGET"`, and the bundle (manifest + contents) is not already in the knowledge object, the node will attempt to retrieve it from the local cache, regardless of the source. In this case the knowledge object represents what we will delete from the cache, not new incoming knowledge. If it fails to retrieve the bundle, the pipeline will end. Next, the bundle handler chain will be called.
|
|
343
|
+
|
|
344
|
+
### Manifest Handler
|
|
345
|
+
The knowledge object passed to handlers of this type are guaranteed to have an RID, manifest, and knowledge source field. This handler type acts as a filter, if none of the handlers return `STOP_CHAIN` the pipeline will progress to the next phase.
|
|
346
|
+
|
|
347
|
+
If the bundle (manifest + contents) is not already in the knowledge object, the node will attempt to retrieve it from (1) the local cache if the source is internal, or (2) from another node if the source is external. If it fails to retrieve the bundle, the pipeline will end. Next, the bundle handler chain will be called.
|
|
348
|
+
|
|
349
|
+
### Bundle Handler
|
|
350
|
+
The knowledge object passed to handlers of this type are guaranteed to have an RID, manifest, bundle (manifest + contents), and knowledge source field. This handler type acts as a decider. In this phase, the knowledge object's normalized event type must be set to `"NEW"` or `"UPDATE"` to write it to cache, or `"FORGET"` to delete it from the cache. If the normalized event type remains unset (`None`), or a handler returns `STOP_CHAIN`, then the pipeline will end without taking any cache action.
|
|
351
|
+
|
|
352
|
+
The cache action will take place after the handler chain ends, so if multiple handlers set a normalized event type, the final handler will take precedence.
|
|
353
|
+
|
|
354
|
+
### Network Handler
|
|
355
|
+
The knowledge object passed to handlers of this type are guaranteed to have an RID, manifest, bundle (manifest + contents), normalized event type, and knowledge source field. This handler type acts as a decider. In this phase, handlers decide which nodes to broadcast this knowledge object to by appending KOI-net node RIDs to the knowledge object's `network_targets` field. If a handler returns `STOP_CHAIN`, the pipeline will end without taking any network action.
|
|
356
|
+
|
|
357
|
+
The network action will take place after the handler chain ends. The node will attempt to broadcast a "normalized event", created from the knowledge object's RID, bundle, and normalized event type, to all of the node's in the network targets array.
|
|
358
|
+
|
|
359
|
+
### Final Handler
|
|
360
|
+
The knowledge object passed to handlers of this type are guaranteed to have an RID, manifest, bundle (manifest + contents), normalized event type, and knowledge source field.
|
|
361
|
+
|
|
362
|
+
This is the final handler chain that is called, it doesn't make any decisions or filter for succesive handler types. Handlers here can be useful if you want to take some action after the network broadcast has ended.
|
|
363
|
+
|
|
364
|
+
## Registering Handlers
|
|
365
|
+
Knowledge handlers are registered with a node's processor by decorating a handler function. There are two types of decorators, the first way converts the function into a handler object which can be manually added to a processor. This is how the default handlers are defined, and makes them more portable (could be imported from another package). The second automatically registers a handler with your node instance. This is not portable but more convenient. The input of the decorated function will be the processor instance, and a knowledge object.
|
|
366
|
+
|
|
367
|
+
```python
|
|
368
|
+
from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
|
|
369
|
+
|
|
370
|
+
@KnowledgeHandler.create(HandlerType.RID)
|
|
371
|
+
def example_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
372
|
+
...
|
|
373
|
+
|
|
374
|
+
@node.processor.register_handler(HandlerType.RID)
|
|
375
|
+
def example_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
376
|
+
...
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
While handler's only require specifying the handler type, you can also specify the RID types, knowledge source, or event types you want to handle. If a knowledge object doesn't match all of the specified parameters, it won't be called. By default, handlers will match all RID types, all event types, and both internal and external sourced knowledge.
|
|
380
|
+
|
|
381
|
+
```python
|
|
382
|
+
@KnowledgeHandler.create(
|
|
383
|
+
handler_type=HandlerType.Bundle,
|
|
384
|
+
rid_types=[KoiNetEdge],
|
|
385
|
+
source=KnowledgeSource.External,
|
|
386
|
+
event_types=[EventType.NEW, EventType.UPDATE])
|
|
387
|
+
def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
388
|
+
...
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
The processor instance passed to your function should be used to take any necessary node actions (cache, network, etc.). It is also sometimes useful to add new knowledge objects to the queue while processing a different knowledge object. You can simply call `processor.handle(...)` in the same way as you would outside of a handler. It will put at the end of the queue and processed when it is dequeued like any other knowledge object.
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
## Default Behavior
|
|
395
|
+
|
|
396
|
+
The default configuration provides four default handlers which will take precedence over any handlers you add yourself. To override this behavior, you can set the `handlers` field in the `NodeInterface`:
|
|
397
|
+
|
|
398
|
+
```python
|
|
399
|
+
from koi_net import NodeInterface
|
|
400
|
+
from koi_net.protocol.node import NodeProfile, NodeProvides, NodeType
|
|
401
|
+
from koi_net.processor.default_handlers import (
|
|
402
|
+
basic_rid_handler,
|
|
403
|
+
basic_manifest_handler,
|
|
404
|
+
edge_negotiation_handler,
|
|
405
|
+
basic_network_output_filter
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
node = NodeInterface(
|
|
409
|
+
name="mypartialnode",
|
|
410
|
+
profile=NodeProfile(
|
|
411
|
+
node_type=NodeType.PARTIAL,
|
|
412
|
+
provides=NodeProvides(
|
|
413
|
+
event=[]
|
|
414
|
+
)
|
|
415
|
+
),
|
|
416
|
+
handlers=[
|
|
417
|
+
basic_rid_handler,
|
|
418
|
+
basic_manifest_handler,
|
|
419
|
+
edge_negotiation_handler,
|
|
420
|
+
basic_network_output_filter
|
|
421
|
+
|
|
422
|
+
# include all or none of the default handlers
|
|
423
|
+
]
|
|
424
|
+
)
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
Take a look at `src/koi_net/processor/default_handlers.py` to see some more in depth examples and better understand the default node behavior.
|
|
428
|
+
|
|
255
429
|
# Implementation Reference
|
|
256
430
|
This section provides high level explanations of the Python implementation. More detailed explanations of methods can be found in the docstrings within the codebase itself.
|
|
257
431
|
|
|
@@ -464,14 +638,13 @@ class ProcessorInterface:
|
|
|
464
638
|
event: Event | None = None,
|
|
465
639
|
kobj: KnowledgeObject | None = None,
|
|
466
640
|
event_type: KnowledgeEventType = None,
|
|
467
|
-
source: KnowledgeSource = KnowledgeSource.Internal
|
|
468
|
-
flush: bool = False
|
|
641
|
+
source: KnowledgeSource = KnowledgeSource.Internal
|
|
469
642
|
): ...
|
|
470
643
|
```
|
|
471
644
|
|
|
472
645
|
The `register_handler` method is a decorator which can wrap a function to create a new `KnowledgeHandler` and add it to the processing pipeline in a single step. The `add_handler` method adds an existing `KnowledgeHandler` to the processining pipeline.
|
|
473
646
|
|
|
474
|
-
The most commonly used functions in this class are `handle` and `flush_kobj_queue`. The `handle` method can be called on RIDs, manifests, bundles, and events to convert them to normalized to `KnowledgeObject` instances which are then added to the processing queue.
|
|
647
|
+
The most commonly used functions in this class are `handle` and `flush_kobj_queue`. The `handle` method can be called on RIDs, manifests, bundles, and events to convert them to normalized to `KnowledgeObject` instances which are then added to the processing queue. If you have enabled `use_kobj_processor_thread` then the queue will be automatically processed, otherwise you will need to regularly call `flush_kobj_queue` to process queued knolwedge objects. When calling the `handle` method, knowledge objects are marked as internally source by default. If you are handling RIDs, manifests, bundles, or events sourced from other nodes, `source` should be set to `KnowledgeSource.External`.
|
|
475
648
|
|
|
476
649
|
Here is an example of how an event polling loop would be implemented using the knowledge processing pipeline:
|
|
477
650
|
```python
|
|
@@ -511,5 +684,5 @@ python -m build
|
|
|
511
684
|
```
|
|
512
685
|
Push new package build to PyPI:
|
|
513
686
|
```shell
|
|
514
|
-
python -m twine upload dist/*
|
|
687
|
+
python -m twine upload --skip-existing dist/*
|
|
515
688
|
```
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
|
|
2
|
+
koi_net/core.py,sha256=9Paw7GTf3JeE8X73fGgi9K2uoT2A20Ft4m_gA9xAepI,4683
|
|
3
|
+
koi_net/identity.py,sha256=PBgmAx5f3zzQmHASB1TJW2g19n9TLfmSJMXg2eQFg0A,2386
|
|
4
|
+
koi_net/network/__init__.py,sha256=r_RN-q_mDYC-2RAkN-lJoMUX76TXyfEUc_MVKW87z0g,39
|
|
5
|
+
koi_net/network/graph.py,sha256=KyS4Vv94DhHQk12u6rmE8nOIuTxeoJjrnoYEwVMNM74,4837
|
|
6
|
+
koi_net/network/interface.py,sha256=8zO2qBYmyd-MyAX-8RAf1mjL_0fn08-IbVOKHCAwNf8,10660
|
|
7
|
+
koi_net/network/request_handler.py,sha256=dufyJRwKD5w578UM-wzFVT3EU78NMvrh_yNSiI7vgfw,4938
|
|
8
|
+
koi_net/network/response_handler.py,sha256=HaP8Fl0bp_lfMmevhdVY8s9o0Uf8CR1ZaW5g3jsX8gw,1888
|
|
9
|
+
koi_net/processor/__init__.py,sha256=x4fAY0hvQEDcpfdTB3POIzxBQjYAtn0qQazPo1Xm0m4,41
|
|
10
|
+
koi_net/processor/default_handlers.py,sha256=ebFJ_MqvGBAb0SxmreZ9Pvyi0tD7ygDvJKECglw8uAQ,6972
|
|
11
|
+
koi_net/processor/handler.py,sha256=a2wxG3kgux-07YP13HSj_PH5VH_PjbnTS4uZEHjxMw0,2582
|
|
12
|
+
koi_net/processor/interface.py,sha256=8DT7-FFWj2vh-DXtuI9NXicOTSozaMSDTziMC2RMwuQ,13010
|
|
13
|
+
koi_net/processor/knowledge_object.py,sha256=xI1K4lIGJrWbb7yStjpU4eqZGI7t4wFklv72V7TRfXY,4212
|
|
14
|
+
koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
koi_net/protocol/api_models.py,sha256=RDwVHAahiWzwzUnlj5MIm9et5WVpQOaG-Uscv1B9coU,1116
|
|
16
|
+
koi_net/protocol/consts.py,sha256=zeWJvRpqcERrqJq39heyNHb6f_9QrvoBZJHd70yE914,249
|
|
17
|
+
koi_net/protocol/edge.py,sha256=G3D9Ie0vbTSMJdoTw9g_oBmFCqzJ1gO7U1PVrw7p3j8,447
|
|
18
|
+
koi_net/protocol/event.py,sha256=YHsLD9HPwib-HiCWv9ep0sPqL2VHQqx_egDmbriChXM,1378
|
|
19
|
+
koi_net/protocol/helpers.py,sha256=9E9PaoIuSNrTBATGCLJ_kSBMZ2z-KIMnLJzGOTqQDC0,719
|
|
20
|
+
koi_net/protocol/node.py,sha256=Ntrx01dbm39ViKGtr4gLmztcMwKpTIweS6rRL-zoU_Y,391
|
|
21
|
+
koi_net-1.0.0b10.dist-info/METADATA,sha256=eQtnFOOH0wpuPMaASpOJJieHhEUadPvVcZ7XdQw0Jh8,33783
|
|
22
|
+
koi_net-1.0.0b10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
23
|
+
koi_net-1.0.0b10.dist-info/licenses/LICENSE,sha256=XBcvl8yjCAezfuqN1jadQykrX7H2g4nr2WRDmHLW6ik,1090
|
|
24
|
+
koi_net-1.0.0b10.dist-info/RECORD,,
|
koi_net-1.0.0b8.dist-info/RECORD
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
|
|
2
|
-
koi_net/core.py,sha256=dE4sE2qsoIRUU1zsnrjx7aqYtYdHyCx-Dv4cwbkRjy4,4613
|
|
3
|
-
koi_net/identity.py,sha256=PBgmAx5f3zzQmHASB1TJW2g19n9TLfmSJMXg2eQFg0A,2386
|
|
4
|
-
koi_net/network/__init__.py,sha256=r_RN-q_mDYC-2RAkN-lJoMUX76TXyfEUc_MVKW87z0g,39
|
|
5
|
-
koi_net/network/graph.py,sha256=KMUCU3AweRvivwy7GuWgX2zX74FPgHeVMO5ydvhVyvA,4833
|
|
6
|
-
koi_net/network/interface.py,sha256=4JTeg8Eah0z5YKhcVKJbCVZw_Ghl_6xfG8aa1I5PCWI,10643
|
|
7
|
-
koi_net/network/request_handler.py,sha256=fhuCDsxI8fZ4p5TntcTZR4mnLrLQ61zDy7Oca3ooFCE,4402
|
|
8
|
-
koi_net/network/response_handler.py,sha256=HaP8Fl0bp_lfMmevhdVY8s9o0Uf8CR1ZaW5g3jsX8gw,1888
|
|
9
|
-
koi_net/processor/__init__.py,sha256=x4fAY0hvQEDcpfdTB3POIzxBQjYAtn0qQazPo1Xm0m4,41
|
|
10
|
-
koi_net/processor/default_handlers.py,sha256=Yc7a9n5sAOYMHzzY59VMXYOxQL-6O9zbMQzd61XbIEs,7184
|
|
11
|
-
koi_net/processor/handler.py,sha256=APCECwU7MFcgP7Vu6UTngs0XIjaXSQ_f8rqy8cH5_rM,2242
|
|
12
|
-
koi_net/processor/interface.py,sha256=szLLeDfMgeqU35F2na-LvzytJ0irpCtR9g0empo4JoI,12169
|
|
13
|
-
koi_net/processor/knowledge_object.py,sha256=cGv33fwNZQMylkhlTaQTbk96FVIVbdOUaBsG06u0m4k,4187
|
|
14
|
-
koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
-
koi_net/protocol/api_models.py,sha256=RDwVHAahiWzwzUnlj5MIm9et5WVpQOaG-Uscv1B9coU,1116
|
|
16
|
-
koi_net/protocol/consts.py,sha256=zeWJvRpqcERrqJq39heyNHb6f_9QrvoBZJHd70yE914,249
|
|
17
|
-
koi_net/protocol/edge.py,sha256=G3D9Ie0vbTSMJdoTw9g_oBmFCqzJ1gO7U1PVrw7p3j8,447
|
|
18
|
-
koi_net/protocol/event.py,sha256=dzJmcHbimo7p5NwH2drccF0vMcAj9oQRj3iZ9Bjf7kg,1275
|
|
19
|
-
koi_net/protocol/helpers.py,sha256=9E9PaoIuSNrTBATGCLJ_kSBMZ2z-KIMnLJzGOTqQDC0,719
|
|
20
|
-
koi_net/protocol/node.py,sha256=Ntrx01dbm39ViKGtr4gLmztcMwKpTIweS6rRL-zoU_Y,391
|
|
21
|
-
koi_net-1.0.0b8.dist-info/METADATA,sha256=-oGUkUtRG4biV7yo7WIl2XiDv4k8wVTV_DbyebOiuoI,21352
|
|
22
|
-
koi_net-1.0.0b8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
23
|
-
koi_net-1.0.0b8.dist-info/licenses/LICENSE,sha256=XBcvl8yjCAezfuqN1jadQykrX7H2g4nr2WRDmHLW6ik,1090
|
|
24
|
-
koi_net-1.0.0b8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|