koi-net 1.0.0b19__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- koi_net/actor.py +60 -0
- koi_net/config.py +44 -18
- koi_net/context.py +63 -0
- koi_net/core.py +152 -84
- koi_net/default_actions.py +15 -0
- koi_net/effector.py +139 -0
- koi_net/identity.py +4 -22
- koi_net/lifecycle.py +104 -0
- koi_net/network/__init__.py +0 -1
- koi_net/network/error_handler.py +50 -0
- koi_net/network/event_queue.py +199 -0
- koi_net/network/graph.py +23 -38
- koi_net/network/request_handler.py +129 -66
- koi_net/network/resolver.py +150 -0
- koi_net/network/response_handler.py +15 -6
- koi_net/poller.py +40 -0
- koi_net/processor/__init__.py +0 -1
- koi_net/processor/default_handlers.py +71 -42
- koi_net/processor/handler.py +3 -7
- koi_net/processor/interface.py +15 -214
- koi_net/processor/knowledge_object.py +10 -17
- koi_net/processor/knowledge_pipeline.py +220 -0
- koi_net/protocol/api_models.py +18 -3
- koi_net/protocol/edge.py +26 -1
- koi_net/protocol/envelope.py +58 -0
- koi_net/protocol/errors.py +23 -0
- koi_net/protocol/event.py +0 -3
- koi_net/protocol/node.py +2 -1
- koi_net/protocol/secure.py +160 -0
- koi_net/secure.py +117 -0
- koi_net/server.py +129 -0
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/METADATA +5 -4
- koi_net-1.1.0.dist-info/RECORD +38 -0
- koi_net/network/interface.py +0 -276
- koi_net/protocol/helpers.py +0 -25
- koi_net-1.0.0b19.dist-info/RECORD +0 -25
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/WHEEL +0 -0
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,29 +1,28 @@
|
|
|
1
1
|
"""Provides implementations of default knowledge handlers."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from rid_lib.ext
|
|
4
|
+
from rid_lib.ext import Bundle
|
|
5
|
+
from rid_lib.ext.utils import sha256_hash
|
|
5
6
|
from rid_lib.types import KoiNetNode, KoiNetEdge
|
|
6
7
|
from koi_net.protocol.node import NodeType
|
|
7
|
-
from .interface import ProcessorInterface
|
|
8
8
|
from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
|
|
9
|
-
from .knowledge_object import KnowledgeObject
|
|
9
|
+
from .knowledge_object import KnowledgeObject
|
|
10
|
+
from ..context import HandlerContext
|
|
10
11
|
from ..protocol.event import Event, EventType
|
|
11
|
-
from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
|
|
12
|
+
from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType, generate_edge_bundle
|
|
12
13
|
from ..protocol.node import NodeProfile
|
|
13
|
-
from ..protocol.helpers import generate_edge_bundle
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
# RID handlers
|
|
18
18
|
|
|
19
19
|
@KnowledgeHandler.create(HandlerType.RID)
|
|
20
|
-
def basic_rid_handler(
|
|
20
|
+
def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
21
21
|
"""Default RID handler.
|
|
22
22
|
|
|
23
23
|
Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
|
|
24
24
|
"""
|
|
25
|
-
if (kobj.rid ==
|
|
26
|
-
kobj.source == KnowledgeSource.External):
|
|
25
|
+
if (kobj.rid == ctx.identity.rid and kobj.source):
|
|
27
26
|
logger.debug("Don't let anyone else tell me who I am!")
|
|
28
27
|
return STOP_CHAIN
|
|
29
28
|
|
|
@@ -34,12 +33,12 @@ def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
34
33
|
# Manifest handlers
|
|
35
34
|
|
|
36
35
|
@KnowledgeHandler.create(HandlerType.Manifest)
|
|
37
|
-
def basic_manifest_handler(
|
|
36
|
+
def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
38
37
|
"""Default manifest handler.
|
|
39
38
|
|
|
40
39
|
Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
|
|
41
40
|
"""
|
|
42
|
-
prev_bundle =
|
|
41
|
+
prev_bundle = ctx.cache.read(kobj.rid)
|
|
43
42
|
|
|
44
43
|
if prev_bundle:
|
|
45
44
|
if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
|
|
@@ -61,36 +60,53 @@ def basic_manifest_handler(processor: ProcessorInterface, kobj: KnowledgeObject)
|
|
|
61
60
|
|
|
62
61
|
# Bundle handlers
|
|
63
62
|
|
|
63
|
+
@KnowledgeHandler.create(
|
|
64
|
+
handler_type=HandlerType.Bundle,
|
|
65
|
+
rid_types=[KoiNetNode],
|
|
66
|
+
event_types=[EventType.NEW, EventType.UPDATE]
|
|
67
|
+
)
|
|
68
|
+
def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
69
|
+
node_profile = kobj.bundle.validate_contents(NodeProfile)
|
|
70
|
+
node_rid: KoiNetNode = kobj.rid
|
|
71
|
+
|
|
72
|
+
if sha256_hash(node_profile.public_key) != node_rid.hash:
|
|
73
|
+
logger.warning(f"Public key hash mismatch for {node_rid!r}!")
|
|
74
|
+
return STOP_CHAIN
|
|
75
|
+
|
|
64
76
|
@KnowledgeHandler.create(
|
|
65
77
|
handler_type=HandlerType.Bundle,
|
|
66
78
|
rid_types=[KoiNetEdge],
|
|
67
|
-
source=KnowledgeSource.External,
|
|
68
79
|
event_types=[EventType.NEW, EventType.UPDATE])
|
|
69
|
-
def edge_negotiation_handler(
|
|
80
|
+
def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
70
81
|
"""Handles basic edge negotiation process.
|
|
71
82
|
|
|
72
83
|
Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
|
|
73
84
|
"""
|
|
85
|
+
|
|
86
|
+
# only respond when source is another node
|
|
87
|
+
if kobj.source is None: return
|
|
74
88
|
|
|
75
|
-
edge_profile =
|
|
89
|
+
edge_profile = kobj.bundle.validate_contents(EdgeProfile)
|
|
76
90
|
|
|
77
91
|
# indicates peer subscribing to me
|
|
78
|
-
if edge_profile.source ==
|
|
92
|
+
if edge_profile.source == ctx.identity.rid:
|
|
79
93
|
if edge_profile.status != EdgeStatus.PROPOSED:
|
|
80
94
|
return
|
|
81
95
|
|
|
82
96
|
logger.debug("Handling edge negotiation")
|
|
83
97
|
|
|
84
98
|
peer_rid = edge_profile.target
|
|
85
|
-
|
|
99
|
+
peer_bundle = ctx.effector.deref(peer_rid)
|
|
86
100
|
|
|
87
|
-
if not
|
|
88
|
-
logger.warning(f"Peer {peer_rid} unknown to me")
|
|
101
|
+
if not peer_bundle:
|
|
102
|
+
logger.warning(f"Peer {peer_rid!r} unknown to me")
|
|
89
103
|
return STOP_CHAIN
|
|
90
104
|
|
|
105
|
+
peer_profile = peer_bundle.validate_contents(NodeProfile)
|
|
106
|
+
|
|
91
107
|
# explicitly provided event RID types and (self) node + edge objects
|
|
92
108
|
provided_events = (
|
|
93
|
-
*
|
|
109
|
+
*ctx.identity.profile.provides.event,
|
|
94
110
|
KoiNetNode, KoiNetEdge
|
|
95
111
|
)
|
|
96
112
|
|
|
@@ -107,7 +123,7 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
107
123
|
|
|
108
124
|
if abort:
|
|
109
125
|
event = Event.from_rid(EventType.FORGET, kobj.rid)
|
|
110
|
-
|
|
126
|
+
ctx.event_queue.push_event_to(event, peer_rid, flush=True)
|
|
111
127
|
return STOP_CHAIN
|
|
112
128
|
|
|
113
129
|
else:
|
|
@@ -116,10 +132,10 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
116
132
|
edge_profile.status = EdgeStatus.APPROVED
|
|
117
133
|
updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
|
|
118
134
|
|
|
119
|
-
|
|
135
|
+
ctx.handle(bundle=updated_bundle, event_type=EventType.UPDATE)
|
|
120
136
|
return
|
|
121
137
|
|
|
122
|
-
elif edge_profile.target ==
|
|
138
|
+
elif edge_profile.target == ctx.identity.rid:
|
|
123
139
|
if edge_profile.status == EdgeStatus.APPROVED:
|
|
124
140
|
logger.debug("Edge approved by other node!")
|
|
125
141
|
|
|
@@ -127,88 +143,88 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
|
|
|
127
143
|
# Network handlers
|
|
128
144
|
|
|
129
145
|
@KnowledgeHandler.create(HandlerType.Network, rid_types=[KoiNetNode])
|
|
130
|
-
def coordinator_contact(
|
|
146
|
+
def coordinator_contact(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
131
147
|
node_profile = kobj.bundle.validate_contents(NodeProfile)
|
|
132
|
-
|
|
148
|
+
|
|
133
149
|
# looking for event provider of nodes
|
|
134
150
|
if KoiNetNode not in node_profile.provides.event:
|
|
135
151
|
return
|
|
136
152
|
|
|
137
153
|
# prevents coordinators from attempting to form a self loop
|
|
138
|
-
if kobj.rid ==
|
|
154
|
+
if kobj.rid == ctx.identity.rid:
|
|
139
155
|
return
|
|
140
156
|
|
|
141
157
|
# already have an edge established
|
|
142
|
-
if
|
|
158
|
+
if ctx.graph.get_edge(
|
|
143
159
|
source=kobj.rid,
|
|
144
|
-
target=
|
|
160
|
+
target=ctx.identity.rid,
|
|
145
161
|
) is not None:
|
|
146
162
|
return
|
|
147
163
|
|
|
148
164
|
logger.info("Identified a coordinator!")
|
|
149
165
|
logger.info("Proposing new edge")
|
|
150
166
|
|
|
151
|
-
if
|
|
167
|
+
if ctx.identity.profile.node_type == NodeType.FULL:
|
|
152
168
|
edge_type = EdgeType.WEBHOOK
|
|
153
169
|
else:
|
|
154
170
|
edge_type = EdgeType.POLL
|
|
155
171
|
|
|
156
172
|
# queued for processing
|
|
157
|
-
|
|
173
|
+
ctx.handle(bundle=generate_edge_bundle(
|
|
158
174
|
source=kobj.rid,
|
|
159
|
-
target=
|
|
175
|
+
target=ctx.identity.rid,
|
|
160
176
|
edge_type=edge_type,
|
|
161
177
|
rid_types=[KoiNetNode]
|
|
162
178
|
))
|
|
163
179
|
|
|
164
180
|
logger.info("Catching up on network state")
|
|
165
181
|
|
|
166
|
-
payload =
|
|
182
|
+
payload = ctx.request_handler.fetch_rids(
|
|
167
183
|
node=kobj.rid,
|
|
168
184
|
rid_types=[KoiNetNode]
|
|
169
185
|
)
|
|
170
186
|
for rid in payload.rids:
|
|
171
|
-
if rid ==
|
|
187
|
+
if rid == ctx.identity.rid:
|
|
172
188
|
logger.info("Skipping myself")
|
|
173
189
|
continue
|
|
174
|
-
if
|
|
175
|
-
logger.info(f"Skipping known RID
|
|
190
|
+
if ctx.cache.exists(rid):
|
|
191
|
+
logger.info(f"Skipping known RID {rid!r}")
|
|
176
192
|
continue
|
|
177
193
|
|
|
178
194
|
# marked as external since we are handling RIDs from another node
|
|
179
195
|
# will fetch remotely instead of checking local cache
|
|
180
|
-
|
|
196
|
+
ctx.handle(rid=rid, source=kobj.rid)
|
|
181
197
|
logger.info("Done")
|
|
182
198
|
|
|
183
199
|
|
|
184
200
|
@KnowledgeHandler.create(HandlerType.Network)
|
|
185
|
-
def basic_network_output_filter(
|
|
201
|
+
def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
186
202
|
"""Default network handler.
|
|
187
203
|
|
|
188
204
|
Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
|
|
189
205
|
|
|
190
206
|
involves_me = False
|
|
191
|
-
if kobj.source
|
|
207
|
+
if kobj.source is None:
|
|
192
208
|
if (type(kobj.rid) == KoiNetNode):
|
|
193
|
-
if (kobj.rid ==
|
|
209
|
+
if (kobj.rid == ctx.identity.rid):
|
|
194
210
|
involves_me = True
|
|
195
211
|
|
|
196
212
|
elif type(kobj.rid) == KoiNetEdge:
|
|
197
213
|
edge_profile = kobj.bundle.validate_contents(EdgeProfile)
|
|
198
214
|
|
|
199
|
-
if edge_profile.source ==
|
|
215
|
+
if edge_profile.source == ctx.identity.rid:
|
|
200
216
|
logger.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
|
|
201
217
|
kobj.network_targets.update([edge_profile.target])
|
|
202
218
|
involves_me = True
|
|
203
219
|
|
|
204
|
-
elif edge_profile.target ==
|
|
220
|
+
elif edge_profile.target == ctx.identity.rid:
|
|
205
221
|
logger.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
|
|
206
222
|
kobj.network_targets.update([edge_profile.source])
|
|
207
223
|
involves_me = True
|
|
208
224
|
|
|
209
|
-
if (type(kobj.rid) in
|
|
225
|
+
if (type(kobj.rid) in ctx.identity.profile.provides.event or involves_me):
|
|
210
226
|
# broadcasts to subscribers if I'm an event provider of this RID type OR it involves me
|
|
211
|
-
subscribers =
|
|
227
|
+
subscribers = ctx.graph.get_neighbors(
|
|
212
228
|
direction="out",
|
|
213
229
|
allowed_type=type(kobj.rid)
|
|
214
230
|
)
|
|
@@ -217,4 +233,17 @@ def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeOb
|
|
|
217
233
|
kobj.network_targets.update(subscribers)
|
|
218
234
|
|
|
219
235
|
return kobj
|
|
220
|
-
|
|
236
|
+
|
|
237
|
+
@KnowledgeHandler.create(HandlerType.Final, rid_types=[KoiNetNode])
|
|
238
|
+
def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
|
|
239
|
+
if kobj.normalized_event_type != EventType.FORGET:
|
|
240
|
+
return
|
|
241
|
+
|
|
242
|
+
for edge_rid in ctx.graph.get_edges():
|
|
243
|
+
edge_bundle = ctx.cache.read(edge_rid)
|
|
244
|
+
if not edge_bundle: continue
|
|
245
|
+
edge_profile = edge_bundle.validate_contents(EdgeProfile)
|
|
246
|
+
|
|
247
|
+
if kobj.rid in (edge_profile.source, edge_profile.target):
|
|
248
|
+
logger.debug("Identified edge with forgotten node")
|
|
249
|
+
ctx.handle(rid=edge_rid, event_type=EventType.FORGET)
|
koi_net/processor/handler.py
CHANGED
|
@@ -2,9 +2,7 @@ from dataclasses import dataclass
|
|
|
2
2
|
from enum import StrEnum
|
|
3
3
|
from typing import Callable
|
|
4
4
|
from rid_lib import RIDType
|
|
5
|
-
|
|
6
5
|
from ..protocol.event import EventType
|
|
7
|
-
from .knowledge_object import KnowledgeSource, KnowledgeEventType
|
|
8
6
|
|
|
9
7
|
|
|
10
8
|
class StopChain:
|
|
@@ -37,23 +35,21 @@ class KnowledgeHandler:
|
|
|
37
35
|
func: Callable
|
|
38
36
|
handler_type: HandlerType
|
|
39
37
|
rid_types: list[RIDType] | None
|
|
40
|
-
|
|
41
|
-
event_types: list[KnowledgeEventType] | None = None
|
|
38
|
+
event_types: list[EventType | None] | None = None
|
|
42
39
|
|
|
43
40
|
@classmethod
|
|
44
41
|
def create(
|
|
45
42
|
cls,
|
|
46
43
|
handler_type: HandlerType,
|
|
47
44
|
rid_types: list[RIDType] | None = None,
|
|
48
|
-
|
|
49
|
-
event_types: list[KnowledgeEventType] | None = None
|
|
45
|
+
event_types: list[EventType | None] | None = None
|
|
50
46
|
):
|
|
51
47
|
"""Special decorator that returns a KnowledgeHandler instead of a function.
|
|
52
48
|
|
|
53
49
|
The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
|
|
54
50
|
"""
|
|
55
51
|
def decorator(func: Callable) -> KnowledgeHandler:
|
|
56
|
-
handler = cls(func, handler_type, rid_types,
|
|
52
|
+
handler = cls(func, handler_type, rid_types, event_types)
|
|
57
53
|
return handler
|
|
58
54
|
return decorator
|
|
59
55
|
|
koi_net/processor/interface.py
CHANGED
|
@@ -1,57 +1,31 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import queue
|
|
3
3
|
import threading
|
|
4
|
-
from
|
|
5
|
-
from rid_lib.
|
|
6
|
-
from rid_lib.
|
|
7
|
-
from rid_lib.types.koi_net_edge import KoiNetEdge
|
|
8
|
-
from rid_lib.types.koi_net_node import KoiNetNode
|
|
9
|
-
from ..identity import NodeIdentity
|
|
10
|
-
from ..network import NetworkInterface
|
|
4
|
+
from rid_lib.core import RID
|
|
5
|
+
from rid_lib.ext import Bundle, Manifest
|
|
6
|
+
from rid_lib.types import KoiNetNode
|
|
11
7
|
from ..protocol.event import Event, EventType
|
|
12
|
-
from
|
|
13
|
-
from .
|
|
14
|
-
|
|
15
|
-
HandlerType,
|
|
16
|
-
STOP_CHAIN,
|
|
17
|
-
StopChain
|
|
18
|
-
)
|
|
19
|
-
from .knowledge_object import (
|
|
20
|
-
KnowledgeObject,
|
|
21
|
-
KnowledgeSource,
|
|
22
|
-
KnowledgeEventType
|
|
23
|
-
)
|
|
8
|
+
from .knowledge_object import KnowledgeObject
|
|
9
|
+
from .knowledge_pipeline import KnowledgePipeline
|
|
10
|
+
|
|
24
11
|
|
|
25
12
|
logger = logging.getLogger(__name__)
|
|
26
13
|
|
|
27
14
|
|
|
28
|
-
class ProcessorInterface
|
|
15
|
+
class ProcessorInterface:
|
|
29
16
|
"""Provides access to this node's knowledge processing pipeline."""
|
|
30
|
-
|
|
31
|
-
config: NodeConfig
|
|
32
|
-
cache: Cache
|
|
33
|
-
network: NetworkInterface
|
|
34
|
-
identity: NodeIdentity
|
|
35
|
-
handlers: list[KnowledgeHandler]
|
|
17
|
+
pipeline: KnowledgePipeline
|
|
36
18
|
kobj_queue: queue.Queue[KnowledgeObject]
|
|
37
19
|
use_kobj_processor_thread: bool
|
|
38
20
|
worker_thread: threading.Thread | None = None
|
|
39
21
|
|
|
40
22
|
def __init__(
|
|
41
23
|
self,
|
|
42
|
-
|
|
43
|
-
cache: Cache,
|
|
44
|
-
network: NetworkInterface,
|
|
45
|
-
identity: NodeIdentity,
|
|
24
|
+
pipeline: KnowledgePipeline,
|
|
46
25
|
use_kobj_processor_thread: bool,
|
|
47
|
-
default_handlers: list[KnowledgeHandler] = []
|
|
48
26
|
):
|
|
49
|
-
self.
|
|
50
|
-
self.cache = cache
|
|
51
|
-
self.network = network
|
|
52
|
-
self.identity = identity
|
|
27
|
+
self.pipeline = pipeline
|
|
53
28
|
self.use_kobj_processor_thread = use_kobj_processor_thread
|
|
54
|
-
self.handlers: list[KnowledgeHandler] = default_handlers
|
|
55
29
|
self.kobj_queue = queue.Queue()
|
|
56
30
|
|
|
57
31
|
if self.use_kobj_processor_thread:
|
|
@@ -59,180 +33,7 @@ class ProcessorInterface():
|
|
|
59
33
|
target=self.kobj_processor_worker,
|
|
60
34
|
daemon=True
|
|
61
35
|
)
|
|
62
|
-
|
|
63
|
-
def add_handler(self, handler: KnowledgeHandler):
|
|
64
|
-
self.handlers.append(handler)
|
|
65
|
-
|
|
66
|
-
def register_handler(
|
|
67
|
-
self,
|
|
68
|
-
handler_type: HandlerType,
|
|
69
|
-
rid_types: list[RIDType] | None = None,
|
|
70
|
-
source: KnowledgeSource | None = None,
|
|
71
|
-
event_types: list[KnowledgeEventType] | None = None
|
|
72
|
-
):
|
|
73
|
-
"""Assigns decorated function as handler for this processor."""
|
|
74
|
-
def decorator(func: Callable) -> Callable:
|
|
75
|
-
handler = KnowledgeHandler(func, handler_type, rid_types, source, event_types)
|
|
76
|
-
self.add_handler(handler)
|
|
77
|
-
return func
|
|
78
|
-
return decorator
|
|
79
|
-
|
|
80
|
-
def call_handler_chain(
|
|
81
|
-
self,
|
|
82
|
-
handler_type: HandlerType,
|
|
83
|
-
kobj: KnowledgeObject
|
|
84
|
-
) -> KnowledgeObject | StopChain:
|
|
85
|
-
"""Calls handlers of provided type, chaining their inputs and outputs together.
|
|
86
|
-
|
|
87
|
-
The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types:
|
|
88
|
-
- `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
|
|
89
|
-
- `None` - to keep the same knowledge object for the next handler in the chain
|
|
90
|
-
- `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
|
|
91
|
-
|
|
92
|
-
Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object.
|
|
93
|
-
"""
|
|
94
|
-
|
|
95
|
-
for handler in self.handlers:
|
|
96
|
-
if handler_type != handler.handler_type:
|
|
97
|
-
continue
|
|
98
|
-
|
|
99
|
-
if handler.rid_types and type(kobj.rid) not in handler.rid_types:
|
|
100
|
-
continue
|
|
101
|
-
|
|
102
|
-
if handler.source and handler.source != kobj.source:
|
|
103
|
-
continue
|
|
104
|
-
|
|
105
|
-
if handler.event_types and kobj.event_type not in handler.event_types:
|
|
106
|
-
continue
|
|
107
|
-
|
|
108
|
-
logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
|
|
109
|
-
resp = handler.func(self, kobj.model_copy())
|
|
110
|
-
|
|
111
|
-
# stops handler chain execution
|
|
112
|
-
if resp is STOP_CHAIN:
|
|
113
|
-
logger.debug(f"Handler chain stopped by {handler.func.__name__}")
|
|
114
|
-
return STOP_CHAIN
|
|
115
|
-
# kobj unmodified
|
|
116
|
-
elif resp is None:
|
|
117
|
-
continue
|
|
118
|
-
# kobj modified by handler
|
|
119
|
-
elif isinstance(resp, KnowledgeObject):
|
|
120
|
-
kobj = resp
|
|
121
|
-
logger.debug(f"Knowledge object modified by {handler.func.__name__}")
|
|
122
|
-
else:
|
|
123
|
-
raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
|
|
124
|
-
|
|
125
|
-
return kobj
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def process_kobj(self, kobj: KnowledgeObject) -> None:
|
|
129
|
-
"""Sends provided knowledge obejct through knowledge processing pipeline.
|
|
130
|
-
|
|
131
|
-
Handler chains are called in between major events in the pipeline, indicated by their handler type. Each handler type is guaranteed to have access to certain knowledge, and may affect a subsequent action in the pipeline. The five handler types are as follows:
|
|
132
|
-
- RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
|
|
133
|
-
- Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
|
|
134
|
-
- Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
|
|
135
|
-
- Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
|
|
136
|
-
- Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
|
|
137
|
-
|
|
138
|
-
The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
|
|
139
|
-
"""
|
|
140
|
-
|
|
141
|
-
logger.debug(f"Handling {kobj!r}")
|
|
142
|
-
kobj = self.call_handler_chain(HandlerType.RID, kobj)
|
|
143
|
-
if kobj is STOP_CHAIN: return
|
|
144
|
-
|
|
145
|
-
if kobj.event_type == EventType.FORGET:
|
|
146
|
-
bundle = self.cache.read(kobj.rid)
|
|
147
|
-
if not bundle:
|
|
148
|
-
logger.debug("Local bundle not found")
|
|
149
|
-
return
|
|
150
|
-
|
|
151
|
-
# the bundle (to be deleted) attached to kobj for downstream analysis
|
|
152
|
-
logger.debug("Adding local bundle (to be deleted) to knowledge object")
|
|
153
|
-
kobj.manifest = bundle.manifest
|
|
154
|
-
kobj.contents = bundle.contents
|
|
155
|
-
|
|
156
|
-
else:
|
|
157
|
-
# attempt to retrieve manifest
|
|
158
|
-
if not kobj.manifest:
|
|
159
|
-
logger.debug("Manifest not found")
|
|
160
|
-
if kobj.source == KnowledgeSource.External:
|
|
161
|
-
logger.debug("Attempting to fetch remote manifest")
|
|
162
|
-
manifest = self.network.fetch_remote_manifest(kobj.rid)
|
|
163
|
-
|
|
164
|
-
elif kobj.source == KnowledgeSource.Internal:
|
|
165
|
-
logger.debug("Attempting to read manifest from cache")
|
|
166
|
-
bundle = self.cache.read(kobj.rid)
|
|
167
|
-
if bundle:
|
|
168
|
-
manifest = bundle.manifest
|
|
169
|
-
else:
|
|
170
|
-
manifest = None
|
|
171
|
-
return
|
|
172
|
-
|
|
173
|
-
if not manifest:
|
|
174
|
-
logger.debug("Failed to find manifest")
|
|
175
|
-
return
|
|
176
|
-
|
|
177
|
-
kobj.manifest = manifest
|
|
178
|
-
|
|
179
|
-
kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
|
|
180
|
-
if kobj is STOP_CHAIN: return
|
|
181
|
-
|
|
182
|
-
# attempt to retrieve bundle
|
|
183
|
-
if not kobj.bundle:
|
|
184
|
-
logger.debug("Bundle not found")
|
|
185
|
-
if kobj.source == KnowledgeSource.External:
|
|
186
|
-
logger.debug("Attempting to fetch remote bundle")
|
|
187
|
-
bundle = self.network.fetch_remote_bundle(kobj.rid)
|
|
188
|
-
|
|
189
|
-
elif kobj.source == KnowledgeSource.Internal:
|
|
190
|
-
logger.debug("Attempting to read bundle from cache")
|
|
191
|
-
bundle = self.cache.read(kobj.rid)
|
|
192
|
-
|
|
193
|
-
if not bundle:
|
|
194
|
-
logger.debug("Failed to find bundle")
|
|
195
|
-
return
|
|
196
|
-
|
|
197
|
-
if kobj.manifest != bundle.manifest:
|
|
198
|
-
logger.warning("Retrieved bundle contains a different manifest")
|
|
199
|
-
|
|
200
|
-
kobj.manifest = bundle.manifest
|
|
201
|
-
kobj.contents = bundle.contents
|
|
202
|
-
|
|
203
|
-
kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
|
|
204
|
-
if kobj is STOP_CHAIN: return
|
|
205
|
-
|
|
206
|
-
if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
|
|
207
|
-
logger.info(f"Writing to cache: {kobj!r}")
|
|
208
|
-
self.cache.write(kobj.bundle)
|
|
209
|
-
|
|
210
|
-
elif kobj.normalized_event_type == EventType.FORGET:
|
|
211
|
-
logger.info(f"Deleting from cache: {kobj!r}")
|
|
212
|
-
self.cache.delete(kobj.rid)
|
|
213
|
-
|
|
214
|
-
else:
|
|
215
|
-
logger.debug("Normalized event type was never set, no cache or network operations will occur")
|
|
216
|
-
return
|
|
217
|
-
|
|
218
|
-
if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
|
|
219
|
-
logger.debug("Change to node or edge, regenerating network graph")
|
|
220
|
-
self.network.graph.generate()
|
|
221
|
-
|
|
222
|
-
kobj = self.call_handler_chain(HandlerType.Network, kobj)
|
|
223
|
-
if kobj is STOP_CHAIN: return
|
|
224
|
-
|
|
225
|
-
if kobj.network_targets:
|
|
226
|
-
logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
|
|
227
|
-
else:
|
|
228
|
-
logger.debug("No network targets set")
|
|
229
|
-
|
|
230
|
-
for node in kobj.network_targets:
|
|
231
|
-
self.network.push_event_to(kobj.normalized_event, node)
|
|
232
|
-
self.network.flush_webhook_queue(node)
|
|
233
|
-
|
|
234
|
-
kobj = self.call_handler_chain(HandlerType.Final, kobj)
|
|
235
|
-
|
|
36
|
+
|
|
236
37
|
def flush_kobj_queue(self):
|
|
237
38
|
"""Flushes all knowledge objects from queue and processes them.
|
|
238
39
|
|
|
@@ -246,7 +47,7 @@ class ProcessorInterface():
|
|
|
246
47
|
logger.debug(f"Dequeued {kobj!r}")
|
|
247
48
|
|
|
248
49
|
try:
|
|
249
|
-
self.
|
|
50
|
+
self.pipeline.process(kobj)
|
|
250
51
|
finally:
|
|
251
52
|
self.kobj_queue.task_done()
|
|
252
53
|
logger.debug("Done")
|
|
@@ -258,7 +59,7 @@ class ProcessorInterface():
|
|
|
258
59
|
logger.debug(f"Dequeued {kobj!r}")
|
|
259
60
|
|
|
260
61
|
try:
|
|
261
|
-
self.
|
|
62
|
+
self.pipeline.process(kobj)
|
|
262
63
|
finally:
|
|
263
64
|
self.kobj_queue.task_done()
|
|
264
65
|
logger.debug("Done")
|
|
@@ -276,8 +77,8 @@ class ProcessorInterface():
|
|
|
276
77
|
bundle: Bundle | None = None,
|
|
277
78
|
event: Event | None = None,
|
|
278
79
|
kobj: KnowledgeObject | None = None,
|
|
279
|
-
event_type:
|
|
280
|
-
source:
|
|
80
|
+
event_type: EventType | None = None,
|
|
81
|
+
source: KoiNetNode | None = None
|
|
281
82
|
):
|
|
282
83
|
"""Queues provided knowledge to be handled by processing pipeline.
|
|
283
84
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
from enum import StrEnum
|
|
2
1
|
from pydantic import BaseModel
|
|
3
2
|
from rid_lib import RID
|
|
4
3
|
from rid_lib.ext import Manifest
|
|
@@ -7,12 +6,6 @@ from rid_lib.types.koi_net_node import KoiNetNode
|
|
|
7
6
|
from ..protocol.event import Event, EventType
|
|
8
7
|
|
|
9
8
|
|
|
10
|
-
type KnowledgeEventType = EventType | None
|
|
11
|
-
|
|
12
|
-
class KnowledgeSource(StrEnum):
|
|
13
|
-
Internal = "INTERNAL"
|
|
14
|
-
External = "EXTERNAL"
|
|
15
|
-
|
|
16
9
|
class KnowledgeObject(BaseModel):
|
|
17
10
|
"""A normalized knowledge representation for internal processing.
|
|
18
11
|
|
|
@@ -29,9 +22,9 @@ class KnowledgeObject(BaseModel):
|
|
|
29
22
|
rid: RID
|
|
30
23
|
manifest: Manifest | None = None
|
|
31
24
|
contents: dict | None = None
|
|
32
|
-
event_type:
|
|
33
|
-
normalized_event_type:
|
|
34
|
-
source:
|
|
25
|
+
event_type: EventType | None = None
|
|
26
|
+
normalized_event_type: EventType | None = None
|
|
27
|
+
source: KoiNetNode | None = None
|
|
35
28
|
network_targets: set[KoiNetNode] = set()
|
|
36
29
|
|
|
37
30
|
def __repr__(self):
|
|
@@ -41,8 +34,8 @@ class KnowledgeObject(BaseModel):
|
|
|
41
34
|
def from_rid(
|
|
42
35
|
cls,
|
|
43
36
|
rid: RID,
|
|
44
|
-
event_type:
|
|
45
|
-
source:
|
|
37
|
+
event_type: EventType | None = None,
|
|
38
|
+
source: KoiNetNode | None = None
|
|
46
39
|
) -> "KnowledgeObject":
|
|
47
40
|
return cls(
|
|
48
41
|
rid=rid,
|
|
@@ -54,8 +47,8 @@ class KnowledgeObject(BaseModel):
|
|
|
54
47
|
def from_manifest(
|
|
55
48
|
cls,
|
|
56
49
|
manifest: Manifest,
|
|
57
|
-
event_type:
|
|
58
|
-
source:
|
|
50
|
+
event_type: EventType | None = None,
|
|
51
|
+
source: KoiNetNode | None = None
|
|
59
52
|
) -> "KnowledgeObject":
|
|
60
53
|
return cls(
|
|
61
54
|
rid=manifest.rid,
|
|
@@ -68,8 +61,8 @@ class KnowledgeObject(BaseModel):
|
|
|
68
61
|
def from_bundle(
|
|
69
62
|
cls,
|
|
70
63
|
bundle: Bundle,
|
|
71
|
-
event_type:
|
|
72
|
-
source:
|
|
64
|
+
event_type: EventType | None = None,
|
|
65
|
+
source: KoiNetNode | None = None
|
|
73
66
|
) -> "KnowledgeObject":
|
|
74
67
|
return cls(
|
|
75
68
|
rid=bundle.rid,
|
|
@@ -83,7 +76,7 @@ class KnowledgeObject(BaseModel):
|
|
|
83
76
|
def from_event(
|
|
84
77
|
cls,
|
|
85
78
|
event: Event,
|
|
86
|
-
source:
|
|
79
|
+
source: KoiNetNode | None = None
|
|
87
80
|
) -> "KnowledgeObject":
|
|
88
81
|
return cls(
|
|
89
82
|
rid=event.rid,
|