koi-net 1.0.0b18__py3-none-any.whl → 1.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- koi_net/config.py +43 -18
- koi_net/context.py +55 -0
- koi_net/core.py +115 -49
- koi_net/default_actions.py +15 -0
- koi_net/effector.py +139 -0
- koi_net/identity.py +4 -22
- koi_net/network/__init__.py +0 -1
- koi_net/network/behavior.py +42 -0
- koi_net/network/error_handler.py +50 -0
- koi_net/network/{interface.py → event_queue.py} +52 -131
- koi_net/network/graph.py +18 -33
- koi_net/network/request_handler.py +112 -66
- koi_net/network/resolver.py +150 -0
- koi_net/network/response_handler.py +11 -3
- koi_net/processor/__init__.py +0 -1
- koi_net/processor/default_handlers.py +57 -41
- koi_net/processor/handler.py +3 -7
- koi_net/processor/interface.py +15 -214
- koi_net/processor/knowledge_object.py +10 -17
- koi_net/processor/knowledge_pipeline.py +220 -0
- koi_net/protocol/api_models.py +7 -1
- koi_net/protocol/edge.py +23 -1
- koi_net/protocol/envelope.py +54 -0
- koi_net/protocol/errors.py +23 -0
- koi_net/protocol/event.py +3 -0
- koi_net/protocol/node.py +2 -1
- koi_net/protocol/secure.py +106 -0
- koi_net/secure.py +117 -0
- {koi_net-1.0.0b18.dist-info → koi_net-1.1.0b1.dist-info}/METADATA +3 -2
- koi_net-1.1.0b1.dist-info/RECORD +35 -0
- koi_net/protocol/helpers.py +0 -25
- koi_net-1.0.0b18.dist-info/RECORD +0 -25
- {koi_net-1.0.0b18.dist-info → koi_net-1.1.0b1.dist-info}/WHEEL +0 -0
- {koi_net-1.0.0b18.dist-info → koi_net-1.1.0b1.dist-info}/licenses/LICENSE +0 -0
koi_net/processor/interface.py
CHANGED
|
@@ -1,57 +1,31 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import queue
|
|
3
3
|
import threading
|
|
4
|
-
from
|
|
5
|
-
from rid_lib.
|
|
6
|
-
from rid_lib.
|
|
7
|
-
from rid_lib.types.koi_net_edge import KoiNetEdge
|
|
8
|
-
from rid_lib.types.koi_net_node import KoiNetNode
|
|
9
|
-
from ..identity import NodeIdentity
|
|
10
|
-
from ..network import NetworkInterface
|
|
4
|
+
from rid_lib.core import RID
|
|
5
|
+
from rid_lib.ext import Bundle, Manifest
|
|
6
|
+
from rid_lib.types import KoiNetNode
|
|
11
7
|
from ..protocol.event import Event, EventType
|
|
12
|
-
from
|
|
13
|
-
from .
|
|
14
|
-
|
|
15
|
-
HandlerType,
|
|
16
|
-
STOP_CHAIN,
|
|
17
|
-
StopChain
|
|
18
|
-
)
|
|
19
|
-
from .knowledge_object import (
|
|
20
|
-
KnowledgeObject,
|
|
21
|
-
KnowledgeSource,
|
|
22
|
-
KnowledgeEventType
|
|
23
|
-
)
|
|
8
|
+
from .knowledge_object import KnowledgeObject
|
|
9
|
+
from .knowledge_pipeline import KnowledgePipeline
|
|
10
|
+
|
|
24
11
|
|
|
25
12
|
logger = logging.getLogger(__name__)
|
|
26
13
|
|
|
27
14
|
|
|
28
|
-
class ProcessorInterface
|
|
15
|
+
class ProcessorInterface:
|
|
29
16
|
"""Provides access to this node's knowledge processing pipeline."""
|
|
30
|
-
|
|
31
|
-
config: NodeConfig
|
|
32
|
-
cache: Cache
|
|
33
|
-
network: NetworkInterface
|
|
34
|
-
identity: NodeIdentity
|
|
35
|
-
handlers: list[KnowledgeHandler]
|
|
17
|
+
pipeline: KnowledgePipeline
|
|
36
18
|
kobj_queue: queue.Queue[KnowledgeObject]
|
|
37
19
|
use_kobj_processor_thread: bool
|
|
38
20
|
worker_thread: threading.Thread | None = None
|
|
39
21
|
|
|
40
22
|
def __init__(
|
|
41
23
|
self,
|
|
42
|
-
|
|
43
|
-
cache: Cache,
|
|
44
|
-
network: NetworkInterface,
|
|
45
|
-
identity: NodeIdentity,
|
|
24
|
+
pipeline: KnowledgePipeline,
|
|
46
25
|
use_kobj_processor_thread: bool,
|
|
47
|
-
default_handlers: list[KnowledgeHandler] = []
|
|
48
26
|
):
|
|
49
|
-
self.
|
|
50
|
-
self.cache = cache
|
|
51
|
-
self.network = network
|
|
52
|
-
self.identity = identity
|
|
27
|
+
self.pipeline = pipeline
|
|
53
28
|
self.use_kobj_processor_thread = use_kobj_processor_thread
|
|
54
|
-
self.handlers: list[KnowledgeHandler] = default_handlers
|
|
55
29
|
self.kobj_queue = queue.Queue()
|
|
56
30
|
|
|
57
31
|
if self.use_kobj_processor_thread:
|
|
@@ -59,180 +33,7 @@ class ProcessorInterface():
|
|
|
59
33
|
target=self.kobj_processor_worker,
|
|
60
34
|
daemon=True
|
|
61
35
|
)
|
|
62
|
-
|
|
63
|
-
def add_handler(self, handler: KnowledgeHandler):
|
|
64
|
-
self.handlers.append(handler)
|
|
65
|
-
|
|
66
|
-
def register_handler(
|
|
67
|
-
self,
|
|
68
|
-
handler_type: HandlerType,
|
|
69
|
-
rid_types: list[RIDType] | None = None,
|
|
70
|
-
source: KnowledgeSource | None = None,
|
|
71
|
-
event_types: list[KnowledgeEventType] | None = None
|
|
72
|
-
):
|
|
73
|
-
"""Assigns decorated function as handler for this processor."""
|
|
74
|
-
def decorator(func: Callable) -> Callable:
|
|
75
|
-
handler = KnowledgeHandler(func, handler_type, rid_types, source, event_types)
|
|
76
|
-
self.add_handler(handler)
|
|
77
|
-
return func
|
|
78
|
-
return decorator
|
|
79
|
-
|
|
80
|
-
def call_handler_chain(
|
|
81
|
-
self,
|
|
82
|
-
handler_type: HandlerType,
|
|
83
|
-
kobj: KnowledgeObject
|
|
84
|
-
) -> KnowledgeObject | StopChain:
|
|
85
|
-
"""Calls handlers of provided type, chaining their inputs and outputs together.
|
|
86
|
-
|
|
87
|
-
The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types:
|
|
88
|
-
- `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
|
|
89
|
-
- `None` - to keep the same knowledge object for the next handler in the chain
|
|
90
|
-
- `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
|
|
91
|
-
|
|
92
|
-
Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object.
|
|
93
|
-
"""
|
|
94
|
-
|
|
95
|
-
for handler in self.handlers:
|
|
96
|
-
if handler_type != handler.handler_type:
|
|
97
|
-
continue
|
|
98
|
-
|
|
99
|
-
if handler.rid_types and type(kobj.rid) not in handler.rid_types:
|
|
100
|
-
continue
|
|
101
|
-
|
|
102
|
-
if handler.source and handler.source != kobj.source:
|
|
103
|
-
continue
|
|
104
|
-
|
|
105
|
-
if handler.event_types and kobj.event_type not in handler.event_types:
|
|
106
|
-
continue
|
|
107
|
-
|
|
108
|
-
logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
|
|
109
|
-
resp = handler.func(self, kobj.model_copy())
|
|
110
|
-
|
|
111
|
-
# stops handler chain execution
|
|
112
|
-
if resp is STOP_CHAIN:
|
|
113
|
-
logger.debug(f"Handler chain stopped by {handler.func.__name__}")
|
|
114
|
-
return STOP_CHAIN
|
|
115
|
-
# kobj unmodified
|
|
116
|
-
elif resp is None:
|
|
117
|
-
continue
|
|
118
|
-
# kobj modified by handler
|
|
119
|
-
elif isinstance(resp, KnowledgeObject):
|
|
120
|
-
kobj = resp
|
|
121
|
-
logger.debug(f"Knowledge object modified by {handler.func.__name__}")
|
|
122
|
-
else:
|
|
123
|
-
raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
|
|
124
|
-
|
|
125
|
-
return kobj
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def process_kobj(self, kobj: KnowledgeObject) -> None:
|
|
129
|
-
"""Sends provided knowledge obejct through knowledge processing pipeline.
|
|
130
|
-
|
|
131
|
-
Handler chains are called in between major events in the pipeline, indicated by their handler type. Each handler type is guaranteed to have access to certain knowledge, and may affect a subsequent action in the pipeline. The five handler types are as follows:
|
|
132
|
-
- RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
|
|
133
|
-
- Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
|
|
134
|
-
- Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
|
|
135
|
-
- Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
|
|
136
|
-
- Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
|
|
137
|
-
|
|
138
|
-
The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
|
|
139
|
-
"""
|
|
140
|
-
|
|
141
|
-
logger.debug(f"Handling {kobj!r}")
|
|
142
|
-
kobj = self.call_handler_chain(HandlerType.RID, kobj)
|
|
143
|
-
if kobj is STOP_CHAIN: return
|
|
144
|
-
|
|
145
|
-
if kobj.event_type == EventType.FORGET:
|
|
146
|
-
bundle = self.cache.read(kobj.rid)
|
|
147
|
-
if not bundle:
|
|
148
|
-
logger.debug("Local bundle not found")
|
|
149
|
-
return
|
|
150
|
-
|
|
151
|
-
# the bundle (to be deleted) attached to kobj for downstream analysis
|
|
152
|
-
logger.debug("Adding local bundle (to be deleted) to knowledge object")
|
|
153
|
-
kobj.manifest = bundle.manifest
|
|
154
|
-
kobj.contents = bundle.contents
|
|
155
|
-
|
|
156
|
-
else:
|
|
157
|
-
# attempt to retrieve manifest
|
|
158
|
-
if not kobj.manifest:
|
|
159
|
-
logger.debug("Manifest not found")
|
|
160
|
-
if kobj.source == KnowledgeSource.External:
|
|
161
|
-
logger.debug("Attempting to fetch remote manifest")
|
|
162
|
-
manifest = self.network.fetch_remote_manifest(kobj.rid)
|
|
163
|
-
|
|
164
|
-
elif kobj.source == KnowledgeSource.Internal:
|
|
165
|
-
logger.debug("Attempting to read manifest from cache")
|
|
166
|
-
bundle = self.cache.read(kobj.rid)
|
|
167
|
-
if bundle:
|
|
168
|
-
manifest = bundle.manifest
|
|
169
|
-
else:
|
|
170
|
-
manifest = None
|
|
171
|
-
return
|
|
172
|
-
|
|
173
|
-
if not manifest:
|
|
174
|
-
logger.debug("Failed to find manifest")
|
|
175
|
-
return
|
|
176
|
-
|
|
177
|
-
kobj.manifest = manifest
|
|
178
|
-
|
|
179
|
-
kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
|
|
180
|
-
if kobj is STOP_CHAIN: return
|
|
181
|
-
|
|
182
|
-
# attempt to retrieve bundle
|
|
183
|
-
if not kobj.bundle:
|
|
184
|
-
logger.debug("Bundle not found")
|
|
185
|
-
if kobj.source == KnowledgeSource.External:
|
|
186
|
-
logger.debug("Attempting to fetch remote bundle")
|
|
187
|
-
bundle = self.network.fetch_remote_bundle(kobj.rid)
|
|
188
|
-
|
|
189
|
-
elif kobj.source == KnowledgeSource.Internal:
|
|
190
|
-
logger.debug("Attempting to read bundle from cache")
|
|
191
|
-
bundle = self.cache.read(kobj.rid)
|
|
192
|
-
|
|
193
|
-
if not bundle:
|
|
194
|
-
logger.debug("Failed to find bundle")
|
|
195
|
-
return
|
|
196
|
-
|
|
197
|
-
if kobj.manifest != bundle.manifest:
|
|
198
|
-
logger.warning("Retrieved bundle contains a different manifest")
|
|
199
|
-
|
|
200
|
-
kobj.manifest = bundle.manifest
|
|
201
|
-
kobj.contents = bundle.contents
|
|
202
|
-
|
|
203
|
-
kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
|
|
204
|
-
if kobj is STOP_CHAIN: return
|
|
205
|
-
|
|
206
|
-
if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
|
|
207
|
-
logger.info(f"Writing to cache: {kobj!r}")
|
|
208
|
-
self.cache.write(kobj.bundle)
|
|
209
|
-
|
|
210
|
-
elif kobj.normalized_event_type == EventType.FORGET:
|
|
211
|
-
logger.info(f"Deleting from cache: {kobj!r}")
|
|
212
|
-
self.cache.delete(kobj.rid)
|
|
213
|
-
|
|
214
|
-
else:
|
|
215
|
-
logger.debug("Normalized event type was never set, no cache or network operations will occur")
|
|
216
|
-
return
|
|
217
|
-
|
|
218
|
-
if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
|
|
219
|
-
logger.debug("Change to node or edge, regenerating network graph")
|
|
220
|
-
self.network.graph.generate()
|
|
221
|
-
|
|
222
|
-
kobj = self.call_handler_chain(HandlerType.Network, kobj)
|
|
223
|
-
if kobj is STOP_CHAIN: return
|
|
224
|
-
|
|
225
|
-
if kobj.network_targets:
|
|
226
|
-
logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
|
|
227
|
-
else:
|
|
228
|
-
logger.debug("No network targets set")
|
|
229
|
-
|
|
230
|
-
for node in kobj.network_targets:
|
|
231
|
-
self.network.push_event_to(kobj.normalized_event, node)
|
|
232
|
-
self.network.flush_webhook_queue(node)
|
|
233
|
-
|
|
234
|
-
kobj = self.call_handler_chain(HandlerType.Final, kobj)
|
|
235
|
-
|
|
36
|
+
|
|
236
37
|
def flush_kobj_queue(self):
|
|
237
38
|
"""Flushes all knowledge objects from queue and processes them.
|
|
238
39
|
|
|
@@ -246,7 +47,7 @@ class ProcessorInterface():
|
|
|
246
47
|
logger.debug(f"Dequeued {kobj!r}")
|
|
247
48
|
|
|
248
49
|
try:
|
|
249
|
-
self.
|
|
50
|
+
self.pipeline.process(kobj)
|
|
250
51
|
finally:
|
|
251
52
|
self.kobj_queue.task_done()
|
|
252
53
|
logger.debug("Done")
|
|
@@ -258,7 +59,7 @@ class ProcessorInterface():
|
|
|
258
59
|
logger.debug(f"Dequeued {kobj!r}")
|
|
259
60
|
|
|
260
61
|
try:
|
|
261
|
-
self.
|
|
62
|
+
self.pipeline.process(kobj)
|
|
262
63
|
finally:
|
|
263
64
|
self.kobj_queue.task_done()
|
|
264
65
|
logger.debug("Done")
|
|
@@ -276,8 +77,8 @@ class ProcessorInterface():
|
|
|
276
77
|
bundle: Bundle | None = None,
|
|
277
78
|
event: Event | None = None,
|
|
278
79
|
kobj: KnowledgeObject | None = None,
|
|
279
|
-
event_type:
|
|
280
|
-
source:
|
|
80
|
+
event_type: EventType | None = None,
|
|
81
|
+
source: KoiNetNode | None = None
|
|
281
82
|
):
|
|
282
83
|
"""Queues provided knowledge to be handled by processing pipeline.
|
|
283
84
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
from enum import StrEnum
|
|
2
1
|
from pydantic import BaseModel
|
|
3
2
|
from rid_lib import RID
|
|
4
3
|
from rid_lib.ext import Manifest
|
|
@@ -7,12 +6,6 @@ from rid_lib.types.koi_net_node import KoiNetNode
|
|
|
7
6
|
from ..protocol.event import Event, EventType
|
|
8
7
|
|
|
9
8
|
|
|
10
|
-
type KnowledgeEventType = EventType | None
|
|
11
|
-
|
|
12
|
-
class KnowledgeSource(StrEnum):
|
|
13
|
-
Internal = "INTERNAL"
|
|
14
|
-
External = "EXTERNAL"
|
|
15
|
-
|
|
16
9
|
class KnowledgeObject(BaseModel):
|
|
17
10
|
"""A normalized knowledge representation for internal processing.
|
|
18
11
|
|
|
@@ -29,9 +22,9 @@ class KnowledgeObject(BaseModel):
|
|
|
29
22
|
rid: RID
|
|
30
23
|
manifest: Manifest | None = None
|
|
31
24
|
contents: dict | None = None
|
|
32
|
-
event_type:
|
|
33
|
-
normalized_event_type:
|
|
34
|
-
source:
|
|
25
|
+
event_type: EventType | None = None
|
|
26
|
+
normalized_event_type: EventType | None = None
|
|
27
|
+
source: KoiNetNode | None = None
|
|
35
28
|
network_targets: set[KoiNetNode] = set()
|
|
36
29
|
|
|
37
30
|
def __repr__(self):
|
|
@@ -41,8 +34,8 @@ class KnowledgeObject(BaseModel):
|
|
|
41
34
|
def from_rid(
|
|
42
35
|
cls,
|
|
43
36
|
rid: RID,
|
|
44
|
-
event_type:
|
|
45
|
-
source:
|
|
37
|
+
event_type: EventType | None = None,
|
|
38
|
+
source: KoiNetNode | None = None
|
|
46
39
|
) -> "KnowledgeObject":
|
|
47
40
|
return cls(
|
|
48
41
|
rid=rid,
|
|
@@ -54,8 +47,8 @@ class KnowledgeObject(BaseModel):
|
|
|
54
47
|
def from_manifest(
|
|
55
48
|
cls,
|
|
56
49
|
manifest: Manifest,
|
|
57
|
-
event_type:
|
|
58
|
-
source:
|
|
50
|
+
event_type: EventType | None = None,
|
|
51
|
+
source: KoiNetNode | None = None
|
|
59
52
|
) -> "KnowledgeObject":
|
|
60
53
|
return cls(
|
|
61
54
|
rid=manifest.rid,
|
|
@@ -68,8 +61,8 @@ class KnowledgeObject(BaseModel):
|
|
|
68
61
|
def from_bundle(
|
|
69
62
|
cls,
|
|
70
63
|
bundle: Bundle,
|
|
71
|
-
event_type:
|
|
72
|
-
source:
|
|
64
|
+
event_type: EventType | None = None,
|
|
65
|
+
source: KoiNetNode | None = None
|
|
73
66
|
) -> "KnowledgeObject":
|
|
74
67
|
return cls(
|
|
75
68
|
rid=bundle.rid,
|
|
@@ -83,7 +76,7 @@ class KnowledgeObject(BaseModel):
|
|
|
83
76
|
def from_event(
|
|
84
77
|
cls,
|
|
85
78
|
event: Event,
|
|
86
|
-
source:
|
|
79
|
+
source: KoiNetNode | None = None
|
|
87
80
|
) -> "KnowledgeObject":
|
|
88
81
|
return cls(
|
|
89
82
|
rid=event.rid,
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Callable
|
|
3
|
+
from rid_lib.core import RIDType
|
|
4
|
+
from rid_lib.types import KoiNetEdge, KoiNetNode
|
|
5
|
+
from rid_lib.ext import Cache
|
|
6
|
+
from ..protocol.event import EventType
|
|
7
|
+
from ..network.request_handler import RequestHandler
|
|
8
|
+
from ..network.event_queue import NetworkEventQueue
|
|
9
|
+
from ..network.graph import NetworkGraph
|
|
10
|
+
from ..identity import NodeIdentity
|
|
11
|
+
from .handler import (
|
|
12
|
+
KnowledgeHandler,
|
|
13
|
+
HandlerType,
|
|
14
|
+
STOP_CHAIN,
|
|
15
|
+
StopChain
|
|
16
|
+
)
|
|
17
|
+
from .knowledge_object import KnowledgeObject
|
|
18
|
+
|
|
19
|
+
from typing import TYPE_CHECKING
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from ..context import HandlerContext
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class KnowledgePipeline:
|
|
27
|
+
handler_context: "HandlerContext"
|
|
28
|
+
cache: Cache
|
|
29
|
+
identity: NodeIdentity
|
|
30
|
+
request_handler: RequestHandler
|
|
31
|
+
event_queue: NetworkEventQueue
|
|
32
|
+
graph: NetworkGraph
|
|
33
|
+
handlers: list[KnowledgeHandler]
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
handler_context: "HandlerContext",
|
|
38
|
+
cache: Cache,
|
|
39
|
+
request_handler: RequestHandler,
|
|
40
|
+
event_queue: NetworkEventQueue,
|
|
41
|
+
graph: NetworkGraph,
|
|
42
|
+
default_handlers: list[KnowledgeHandler] = []
|
|
43
|
+
):
|
|
44
|
+
self.handler_context = handler_context
|
|
45
|
+
self.cache = cache
|
|
46
|
+
self.request_handler = request_handler
|
|
47
|
+
self.event_queue = event_queue
|
|
48
|
+
self.graph = graph
|
|
49
|
+
self.handlers = default_handlers
|
|
50
|
+
|
|
51
|
+
def add_handler(self, handler: KnowledgeHandler):
|
|
52
|
+
self.handlers.append(handler)
|
|
53
|
+
|
|
54
|
+
def register_handler(
|
|
55
|
+
self,
|
|
56
|
+
handler_type: HandlerType,
|
|
57
|
+
rid_types: list[RIDType] | None = None,
|
|
58
|
+
event_types: list[EventType | None] | None = None
|
|
59
|
+
):
|
|
60
|
+
"""Assigns decorated function as handler for this processor."""
|
|
61
|
+
def decorator(func: Callable) -> Callable:
|
|
62
|
+
handler = KnowledgeHandler(func, handler_type, rid_types, event_types)
|
|
63
|
+
self.add_handler(handler)
|
|
64
|
+
return func
|
|
65
|
+
return decorator
|
|
66
|
+
|
|
67
|
+
def call_handler_chain(
|
|
68
|
+
self,
|
|
69
|
+
handler_type: HandlerType,
|
|
70
|
+
kobj: KnowledgeObject
|
|
71
|
+
) -> KnowledgeObject | StopChain:
|
|
72
|
+
"""Calls handlers of provided type, chaining their inputs and outputs together.
|
|
73
|
+
|
|
74
|
+
The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types:
|
|
75
|
+
- `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
|
|
76
|
+
- `None` - to keep the same knowledge object for the next handler in the chain
|
|
77
|
+
- `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
|
|
78
|
+
|
|
79
|
+
Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
for handler in self.handlers:
|
|
83
|
+
if handler_type != handler.handler_type:
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
if handler.rid_types and type(kobj.rid) not in handler.rid_types:
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
if handler.event_types and kobj.event_type not in handler.event_types:
|
|
90
|
+
continue
|
|
91
|
+
|
|
92
|
+
logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
|
|
93
|
+
|
|
94
|
+
resp = handler.func(
|
|
95
|
+
ctx=self.handler_context,
|
|
96
|
+
kobj=kobj.model_copy()
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# stops handler chain execution
|
|
100
|
+
if resp is STOP_CHAIN:
|
|
101
|
+
logger.debug(f"Handler chain stopped by {handler.func.__name__}")
|
|
102
|
+
return STOP_CHAIN
|
|
103
|
+
# kobj unmodified
|
|
104
|
+
elif resp is None:
|
|
105
|
+
continue
|
|
106
|
+
# kobj modified by handler
|
|
107
|
+
elif isinstance(resp, KnowledgeObject):
|
|
108
|
+
kobj = resp
|
|
109
|
+
logger.debug(f"Knowledge object modified by {handler.func.__name__}")
|
|
110
|
+
else:
|
|
111
|
+
raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
|
|
112
|
+
|
|
113
|
+
return kobj
|
|
114
|
+
|
|
115
|
+
def process(self, kobj: KnowledgeObject):
|
|
116
|
+
"""Sends provided knowledge obejct through knowledge processing pipeline.
|
|
117
|
+
|
|
118
|
+
Handler chains are called in between major events in the pipeline, indicated by their handler type. Each handler type is guaranteed to have access to certain knowledge, and may affect a subsequent action in the pipeline. The five handler types are as follows:
|
|
119
|
+
- RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
|
|
120
|
+
- Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
|
|
121
|
+
- Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
|
|
122
|
+
- Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
|
|
123
|
+
- Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
|
|
124
|
+
|
|
125
|
+
The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
logger.debug(f"Handling {kobj!r}")
|
|
129
|
+
kobj = self.call_handler_chain(HandlerType.RID, kobj)
|
|
130
|
+
if kobj is STOP_CHAIN: return
|
|
131
|
+
|
|
132
|
+
if kobj.event_type == EventType.FORGET:
|
|
133
|
+
bundle = self.cache.read(kobj.rid)
|
|
134
|
+
if not bundle:
|
|
135
|
+
logger.debug("Local bundle not found")
|
|
136
|
+
return
|
|
137
|
+
|
|
138
|
+
# the bundle (to be deleted) attached to kobj for downstream analysis
|
|
139
|
+
logger.debug("Adding local bundle (to be deleted) to knowledge object")
|
|
140
|
+
kobj.manifest = bundle.manifest
|
|
141
|
+
kobj.contents = bundle.contents
|
|
142
|
+
|
|
143
|
+
else:
|
|
144
|
+
# attempt to retrieve manifest
|
|
145
|
+
if not kobj.manifest:
|
|
146
|
+
logger.debug("Manifest not found")
|
|
147
|
+
if not kobj.source:
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
logger.debug("Attempting to fetch remote manifest from source")
|
|
151
|
+
payload = self.request_handler.fetch_manifests(
|
|
152
|
+
node=kobj.source,
|
|
153
|
+
rids=[kobj.rid]
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
if not payload.manifests:
|
|
157
|
+
logger.debug("Failed to find manifest")
|
|
158
|
+
return
|
|
159
|
+
|
|
160
|
+
kobj.manifest = payload.manifests[0]
|
|
161
|
+
|
|
162
|
+
kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
|
|
163
|
+
if kobj is STOP_CHAIN: return
|
|
164
|
+
|
|
165
|
+
# attempt to retrieve bundle
|
|
166
|
+
if not kobj.bundle:
|
|
167
|
+
logger.debug("Bundle not found")
|
|
168
|
+
if kobj.source is None:
|
|
169
|
+
return
|
|
170
|
+
|
|
171
|
+
logger.debug("Attempting to fetch remote bundle from source")
|
|
172
|
+
payload = self.request_handler.fetch_bundles(
|
|
173
|
+
node=kobj.source,
|
|
174
|
+
rids=[kobj.rid]
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if not payload.bundles:
|
|
178
|
+
logger.debug("Failed to find bundle")
|
|
179
|
+
return
|
|
180
|
+
|
|
181
|
+
bundle = payload.bundles[0]
|
|
182
|
+
|
|
183
|
+
if kobj.manifest != bundle.manifest:
|
|
184
|
+
logger.warning("Retrieved bundle contains a different manifest")
|
|
185
|
+
|
|
186
|
+
kobj.manifest = bundle.manifest
|
|
187
|
+
kobj.contents = bundle.contents
|
|
188
|
+
|
|
189
|
+
kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
|
|
190
|
+
if kobj is STOP_CHAIN: return
|
|
191
|
+
|
|
192
|
+
if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
|
|
193
|
+
logger.info(f"Writing to cache: {kobj!r}")
|
|
194
|
+
self.cache.write(kobj.bundle)
|
|
195
|
+
|
|
196
|
+
elif kobj.normalized_event_type == EventType.FORGET:
|
|
197
|
+
logger.info(f"Deleting from cache: {kobj!r}")
|
|
198
|
+
self.cache.delete(kobj.rid)
|
|
199
|
+
|
|
200
|
+
else:
|
|
201
|
+
logger.debug("Normalized event type was never set, no cache or network operations will occur")
|
|
202
|
+
return
|
|
203
|
+
|
|
204
|
+
if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
|
|
205
|
+
logger.debug("Change to node or edge, regenerating network graph")
|
|
206
|
+
self.graph.generate()
|
|
207
|
+
|
|
208
|
+
kobj = self.call_handler_chain(HandlerType.Network, kobj)
|
|
209
|
+
if kobj is STOP_CHAIN: return
|
|
210
|
+
|
|
211
|
+
if kobj.network_targets:
|
|
212
|
+
logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
|
|
213
|
+
else:
|
|
214
|
+
logger.debug("No network targets set")
|
|
215
|
+
|
|
216
|
+
for node in kobj.network_targets:
|
|
217
|
+
self.event_queue.push_event_to(kobj.normalized_event, node)
|
|
218
|
+
self.event_queue.flush_webhook_queue(node)
|
|
219
|
+
|
|
220
|
+
kobj = self.call_handler_chain(HandlerType.Final, kobj)
|
koi_net/protocol/api_models.py
CHANGED
|
@@ -4,6 +4,7 @@ from pydantic import BaseModel
|
|
|
4
4
|
from rid_lib import RID, RIDType
|
|
5
5
|
from rid_lib.ext import Bundle, Manifest
|
|
6
6
|
from .event import Event
|
|
7
|
+
from .errors import ErrorTypes
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
# REQUEST MODELS
|
|
@@ -41,7 +42,12 @@ class EventsPayload(BaseModel):
|
|
|
41
42
|
events: list[Event]
|
|
42
43
|
|
|
43
44
|
|
|
45
|
+
# ERROR MODELS
|
|
46
|
+
|
|
47
|
+
class ErrorResponse(BaseModel):
|
|
48
|
+
error: ErrorTypes
|
|
49
|
+
|
|
44
50
|
# TYPES
|
|
45
51
|
|
|
46
52
|
type RequestModels = EventsPayload | PollEvents | FetchRids | FetchManifests | FetchBundles
|
|
47
|
-
type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload
|
|
53
|
+
type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload | ErrorResponse
|
koi_net/protocol/edge.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from enum import StrEnum
|
|
2
2
|
from pydantic import BaseModel
|
|
3
3
|
from rid_lib import RIDType
|
|
4
|
-
from rid_lib.
|
|
4
|
+
from rid_lib.ext.bundle import Bundle
|
|
5
|
+
from rid_lib.types import KoiNetEdge, KoiNetNode
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
class EdgeStatus(StrEnum):
|
|
@@ -18,3 +19,24 @@ class EdgeProfile(BaseModel):
|
|
|
18
19
|
edge_type: EdgeType
|
|
19
20
|
status: EdgeStatus
|
|
20
21
|
rid_types: list[RIDType]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def generate_edge_bundle(
|
|
25
|
+
source: KoiNetNode,
|
|
26
|
+
target: KoiNetNode,
|
|
27
|
+
rid_types: list[RIDType],
|
|
28
|
+
edge_type: EdgeType
|
|
29
|
+
) -> Bundle:
|
|
30
|
+
edge_rid = KoiNetEdge.generate(source, target)
|
|
31
|
+
edge_profile = EdgeProfile(
|
|
32
|
+
source=source,
|
|
33
|
+
target=target,
|
|
34
|
+
rid_types=rid_types,
|
|
35
|
+
edge_type=edge_type,
|
|
36
|
+
status=EdgeStatus.PROPOSED
|
|
37
|
+
)
|
|
38
|
+
edge_bundle = Bundle.generate(
|
|
39
|
+
edge_rid,
|
|
40
|
+
edge_profile.model_dump()
|
|
41
|
+
)
|
|
42
|
+
return edge_bundle
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Generic, TypeVar
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
from rid_lib.types import KoiNetNode
|
|
5
|
+
|
|
6
|
+
from .secure import PrivateKey, PublicKey
|
|
7
|
+
from .api_models import RequestModels, ResponseModels
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
T = TypeVar("T", bound=RequestModels | ResponseModels)
|
|
14
|
+
|
|
15
|
+
class SignedEnvelope(BaseModel, Generic[T]):
|
|
16
|
+
payload: T
|
|
17
|
+
source_node: KoiNetNode
|
|
18
|
+
target_node: KoiNetNode
|
|
19
|
+
signature: str
|
|
20
|
+
|
|
21
|
+
def verify_with(self, pub_key: PublicKey):
|
|
22
|
+
# IMPORTANT: calling `model_dump()` loses all typing! when converting between SignedEnvelope and UnsignedEnvelope, use the Pydantic classes, not the dictionary form
|
|
23
|
+
unsigned_envelope = UnsignedEnvelope[T](
|
|
24
|
+
payload=self.payload,
|
|
25
|
+
source_node=self.source_node,
|
|
26
|
+
target_node=self.target_node
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
logger.debug(f"Verifying envelope: {unsigned_envelope.model_dump_json()}")
|
|
30
|
+
|
|
31
|
+
pub_key.verify(
|
|
32
|
+
self.signature,
|
|
33
|
+
unsigned_envelope.model_dump_json().encode()
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
class UnsignedEnvelope(BaseModel, Generic[T]):
|
|
37
|
+
payload: T
|
|
38
|
+
source_node: KoiNetNode
|
|
39
|
+
target_node: KoiNetNode
|
|
40
|
+
|
|
41
|
+
def sign_with(self, priv_key: PrivateKey) -> SignedEnvelope[T]:
|
|
42
|
+
logger.debug(f"Signing envelope: {self.model_dump_json()}")
|
|
43
|
+
logger.debug(f"Type: [{type(self.payload)}]")
|
|
44
|
+
|
|
45
|
+
signature = priv_key.sign(
|
|
46
|
+
self.model_dump_json().encode()
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
return SignedEnvelope(
|
|
50
|
+
payload=self.payload,
|
|
51
|
+
source_node=self.source_node,
|
|
52
|
+
target_node=self.target_node,
|
|
53
|
+
signature=signature
|
|
54
|
+
)
|