koi-net 1.0.0b6__tar.gz → 1.0.0b8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/PKG-INFO +19 -14
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/README.md +18 -13
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/basic_coordinator_node.py +8 -7
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/basic_partial_node.py +5 -3
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/full_node_template.py +5 -6
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/partial_node_template.py +2 -2
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/pyproject.toml +1 -1
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/core.py +36 -12
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/graph.py +3 -3
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/interface.py +4 -7
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/response_handler.py +1 -1
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/interface.py +39 -12
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/api_models.py +1 -1
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/.gitignore +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/LICENSE +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/requirements.txt +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/__init__.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/identity.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/__init__.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/request_handler.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/__init__.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/default_handlers.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/handler.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/knowledge_object.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/__init__.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/consts.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/edge.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/event.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/helpers.py +0 -0
- {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/node.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: koi-net
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.0b8
|
|
4
4
|
Summary: Implementation of KOI-net protocol in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/BlockScience/koi-net/
|
|
6
6
|
Author-email: Luke Miller <luke@block.science>
|
|
@@ -144,13 +144,14 @@ node = NodeInterface(
|
|
|
144
144
|
event=[],
|
|
145
145
|
state=[]
|
|
146
146
|
)
|
|
147
|
-
)
|
|
147
|
+
),
|
|
148
|
+
use_kobj_processor_thread=True
|
|
148
149
|
)
|
|
149
150
|
```
|
|
150
151
|
|
|
151
152
|
## Knowledge Processing
|
|
152
153
|
|
|
153
|
-
Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.
|
|
154
|
+
Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.start()` and `node.stop()` at the beginning and end of your node's life cycle.
|
|
154
155
|
|
|
155
156
|
### Partial Node
|
|
156
157
|
Make sure to set `source=KnowledgeSource.External`, this indicates to the knowledge processing pipeline that the incoming knowledge was received from an external source. Where the knowledge is sourced from will impact decisions in the node's knowledge handlers.
|
|
@@ -159,7 +160,7 @@ import time
|
|
|
159
160
|
from koi_net.processor.knowledge_object import KnowledgeSource
|
|
160
161
|
|
|
161
162
|
if __name__ == "__main__":
|
|
162
|
-
node.
|
|
163
|
+
node.start()
|
|
163
164
|
|
|
164
165
|
try:
|
|
165
166
|
while True:
|
|
@@ -170,20 +171,20 @@ if __name__ == "__main__":
|
|
|
170
171
|
time.sleep(5)
|
|
171
172
|
|
|
172
173
|
finally:
|
|
173
|
-
node.
|
|
174
|
+
node.stop()
|
|
174
175
|
```
|
|
175
176
|
|
|
176
177
|
### Full Node
|
|
177
|
-
Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to
|
|
178
|
+
Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to start and stop the node before and after execution, as well as the FastAPI app which will be our web server.
|
|
178
179
|
```python
|
|
179
180
|
from contextlib import asynccontextmanager
|
|
180
181
|
from fastapi import FastAPI
|
|
181
182
|
|
|
182
183
|
@asynccontextmanager
|
|
183
184
|
async def lifespan(app: FastAPI):
|
|
184
|
-
node.
|
|
185
|
+
node.start()
|
|
185
186
|
yield
|
|
186
|
-
node.
|
|
187
|
+
node.stop()
|
|
187
188
|
|
|
188
189
|
|
|
189
190
|
app = FastAPI(lifespan=lifespan, root_path="/koi-net")
|
|
@@ -196,11 +197,9 @@ from koi_net.protocol.api_models import *
|
|
|
196
197
|
from koi_net.protocol.consts import *
|
|
197
198
|
|
|
198
199
|
@app.post(BROADCAST_EVENTS_PATH)
|
|
199
|
-
def broadcast_events(req: EventsPayload
|
|
200
|
+
def broadcast_events(req: EventsPayload):
|
|
200
201
|
for event in req.events:
|
|
201
202
|
node.processor.handle(event=event, source=KnowledgeSource.External)
|
|
202
|
-
|
|
203
|
-
background.add_task(node.processor.flush_kobj_queue)
|
|
204
203
|
```
|
|
205
204
|
|
|
206
205
|
Next we can add the event polling endpoint, this allows partial nodes to receive events from us.
|
|
@@ -265,12 +264,16 @@ class NodeInterface:
|
|
|
265
264
|
network: NetworkInterface
|
|
266
265
|
processor: ProcessorInterface
|
|
267
266
|
first_contact: str
|
|
267
|
+
use_kobj_processor_thread: bool
|
|
268
268
|
|
|
269
269
|
def __init__(
|
|
270
270
|
self,
|
|
271
271
|
name: str,
|
|
272
272
|
profile: NodeProfile,
|
|
273
273
|
identity_file_path: str = "identity.json",
|
|
274
|
+
event_queues_file_path: str = "event_queues.json",
|
|
275
|
+
cache_directory_path: str = "rid_cache",
|
|
276
|
+
use_kobj_processor_thread: bool = False,
|
|
274
277
|
first_contact: str | None = None,
|
|
275
278
|
handlers: list[KnowledgeHandler] | None = None,
|
|
276
279
|
cache: Cache | None = None,
|
|
@@ -278,8 +281,8 @@ class NodeInterface:
|
|
|
278
281
|
processor: ProcessorInterface | None = None
|
|
279
282
|
): ...
|
|
280
283
|
|
|
281
|
-
def
|
|
282
|
-
def
|
|
284
|
+
def start(self): ...
|
|
285
|
+
def stop(self): ...
|
|
283
286
|
```
|
|
284
287
|
As you can see, only a name and profile are required. The other fields allow for additional customization if needed.
|
|
285
288
|
|
|
@@ -313,7 +316,6 @@ class NetworkInterface:
|
|
|
313
316
|
|
|
314
317
|
def flush_poll_queue(self, node: KoiNetNode) -> list[Event]: ...
|
|
315
318
|
def flush_webhook_queue(self, node: RID): ...
|
|
316
|
-
def flush_all_webhook_queues(self): ...
|
|
317
319
|
|
|
318
320
|
def fetch_remote_bundle(self, rid: RID): ...
|
|
319
321
|
def fetch_remote_manifest(self, rid: RID): ...
|
|
@@ -432,11 +434,14 @@ def poll_events(req: PollEvents) -> EventsPayload:
|
|
|
432
434
|
The `ProcessorInterface` class provides access to a node's internal knowledge processing pipeline.
|
|
433
435
|
```python
|
|
434
436
|
class ProcessorInterface:
|
|
437
|
+
worker_thread: threading.Thread | None = None
|
|
438
|
+
|
|
435
439
|
def __init__(
|
|
436
440
|
self,
|
|
437
441
|
cache: Cache,
|
|
438
442
|
network: NetworkInterface,
|
|
439
443
|
identity: NodeIdentity,
|
|
444
|
+
use_kobj_processor_thread: bool,
|
|
440
445
|
default_handlers: list[KnowledgeHandler] = []
|
|
441
446
|
): ...
|
|
442
447
|
|
|
@@ -102,13 +102,14 @@ node = NodeInterface(
|
|
|
102
102
|
event=[],
|
|
103
103
|
state=[]
|
|
104
104
|
)
|
|
105
|
-
)
|
|
105
|
+
),
|
|
106
|
+
use_kobj_processor_thread=True
|
|
106
107
|
)
|
|
107
108
|
```
|
|
108
109
|
|
|
109
110
|
## Knowledge Processing
|
|
110
111
|
|
|
111
|
-
Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.
|
|
112
|
+
Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.start()` and `node.stop()` at the beginning and end of your node's life cycle.
|
|
112
113
|
|
|
113
114
|
### Partial Node
|
|
114
115
|
Make sure to set `source=KnowledgeSource.External`, this indicates to the knowledge processing pipeline that the incoming knowledge was received from an external source. Where the knowledge is sourced from will impact decisions in the node's knowledge handlers.
|
|
@@ -117,7 +118,7 @@ import time
|
|
|
117
118
|
from koi_net.processor.knowledge_object import KnowledgeSource
|
|
118
119
|
|
|
119
120
|
if __name__ == "__main__":
|
|
120
|
-
node.
|
|
121
|
+
node.start()
|
|
121
122
|
|
|
122
123
|
try:
|
|
123
124
|
while True:
|
|
@@ -128,20 +129,20 @@ if __name__ == "__main__":
|
|
|
128
129
|
time.sleep(5)
|
|
129
130
|
|
|
130
131
|
finally:
|
|
131
|
-
node.
|
|
132
|
+
node.stop()
|
|
132
133
|
```
|
|
133
134
|
|
|
134
135
|
### Full Node
|
|
135
|
-
Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to
|
|
136
|
+
Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to start and stop the node before and after execution, as well as the FastAPI app which will be our web server.
|
|
136
137
|
```python
|
|
137
138
|
from contextlib import asynccontextmanager
|
|
138
139
|
from fastapi import FastAPI
|
|
139
140
|
|
|
140
141
|
@asynccontextmanager
|
|
141
142
|
async def lifespan(app: FastAPI):
|
|
142
|
-
node.
|
|
143
|
+
node.start()
|
|
143
144
|
yield
|
|
144
|
-
node.
|
|
145
|
+
node.stop()
|
|
145
146
|
|
|
146
147
|
|
|
147
148
|
app = FastAPI(lifespan=lifespan, root_path="/koi-net")
|
|
@@ -154,11 +155,9 @@ from koi_net.protocol.api_models import *
|
|
|
154
155
|
from koi_net.protocol.consts import *
|
|
155
156
|
|
|
156
157
|
@app.post(BROADCAST_EVENTS_PATH)
|
|
157
|
-
def broadcast_events(req: EventsPayload
|
|
158
|
+
def broadcast_events(req: EventsPayload):
|
|
158
159
|
for event in req.events:
|
|
159
160
|
node.processor.handle(event=event, source=KnowledgeSource.External)
|
|
160
|
-
|
|
161
|
-
background.add_task(node.processor.flush_kobj_queue)
|
|
162
161
|
```
|
|
163
162
|
|
|
164
163
|
Next we can add the event polling endpoint, this allows partial nodes to receive events from us.
|
|
@@ -223,12 +222,16 @@ class NodeInterface:
|
|
|
223
222
|
network: NetworkInterface
|
|
224
223
|
processor: ProcessorInterface
|
|
225
224
|
first_contact: str
|
|
225
|
+
use_kobj_processor_thread: bool
|
|
226
226
|
|
|
227
227
|
def __init__(
|
|
228
228
|
self,
|
|
229
229
|
name: str,
|
|
230
230
|
profile: NodeProfile,
|
|
231
231
|
identity_file_path: str = "identity.json",
|
|
232
|
+
event_queues_file_path: str = "event_queues.json",
|
|
233
|
+
cache_directory_path: str = "rid_cache",
|
|
234
|
+
use_kobj_processor_thread: bool = False,
|
|
232
235
|
first_contact: str | None = None,
|
|
233
236
|
handlers: list[KnowledgeHandler] | None = None,
|
|
234
237
|
cache: Cache | None = None,
|
|
@@ -236,8 +239,8 @@ class NodeInterface:
|
|
|
236
239
|
processor: ProcessorInterface | None = None
|
|
237
240
|
): ...
|
|
238
241
|
|
|
239
|
-
def
|
|
240
|
-
def
|
|
242
|
+
def start(self): ...
|
|
243
|
+
def stop(self): ...
|
|
241
244
|
```
|
|
242
245
|
As you can see, only a name and profile are required. The other fields allow for additional customization if needed.
|
|
243
246
|
|
|
@@ -271,7 +274,6 @@ class NetworkInterface:
|
|
|
271
274
|
|
|
272
275
|
def flush_poll_queue(self, node: KoiNetNode) -> list[Event]: ...
|
|
273
276
|
def flush_webhook_queue(self, node: RID): ...
|
|
274
|
-
def flush_all_webhook_queues(self): ...
|
|
275
277
|
|
|
276
278
|
def fetch_remote_bundle(self, rid: RID): ...
|
|
277
279
|
def fetch_remote_manifest(self, rid: RID): ...
|
|
@@ -390,11 +392,14 @@ def poll_events(req: PollEvents) -> EventsPayload:
|
|
|
390
392
|
The `ProcessorInterface` class provides access to a node's internal knowledge processing pipeline.
|
|
391
393
|
```python
|
|
392
394
|
class ProcessorInterface:
|
|
395
|
+
worker_thread: threading.Thread | None = None
|
|
396
|
+
|
|
393
397
|
def __init__(
|
|
394
398
|
self,
|
|
395
399
|
cache: Cache,
|
|
396
400
|
network: NetworkInterface,
|
|
397
401
|
identity: NodeIdentity,
|
|
402
|
+
use_kobj_processor_thread: bool,
|
|
398
403
|
default_handlers: list[KnowledgeHandler] = []
|
|
399
404
|
): ...
|
|
400
405
|
|
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
import uvicorn
|
|
4
4
|
from contextlib import asynccontextmanager
|
|
5
5
|
from rich.logging import RichHandler
|
|
6
|
-
from fastapi import FastAPI
|
|
6
|
+
from fastapi import FastAPI
|
|
7
7
|
from rid_lib.types import KoiNetNode, KoiNetEdge
|
|
8
8
|
from koi_net import NodeInterface
|
|
9
9
|
from koi_net.processor.handler import HandlerType
|
|
@@ -53,7 +53,10 @@ node = NodeInterface(
|
|
|
53
53
|
state=[KoiNetNode, KoiNetEdge]
|
|
54
54
|
)
|
|
55
55
|
),
|
|
56
|
-
|
|
56
|
+
use_kobj_processor_thread=True,
|
|
57
|
+
cache_directory_path="coordinator_node_rid_cache",
|
|
58
|
+
event_queues_file_path="coordinator_node_event_queus.json",
|
|
59
|
+
identity_file_path="coordinator_node_identity.json",
|
|
57
60
|
)
|
|
58
61
|
|
|
59
62
|
|
|
@@ -88,9 +91,9 @@ def handshake_handler(proc: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
88
91
|
|
|
89
92
|
@asynccontextmanager
|
|
90
93
|
async def lifespan(app: FastAPI):
|
|
91
|
-
node.
|
|
94
|
+
node.start()
|
|
92
95
|
yield
|
|
93
|
-
node.
|
|
96
|
+
node.stop()
|
|
94
97
|
|
|
95
98
|
app = FastAPI(
|
|
96
99
|
lifespan=lifespan,
|
|
@@ -100,13 +103,11 @@ app = FastAPI(
|
|
|
100
103
|
)
|
|
101
104
|
|
|
102
105
|
@app.post(BROADCAST_EVENTS_PATH)
|
|
103
|
-
def broadcast_events(req: EventsPayload
|
|
106
|
+
def broadcast_events(req: EventsPayload):
|
|
104
107
|
logger.info(f"Request to {BROADCAST_EVENTS_PATH}, received {len(req.events)} event(s)")
|
|
105
108
|
for event in req.events:
|
|
106
109
|
node.processor.handle(event=event, source=KnowledgeSource.External)
|
|
107
110
|
|
|
108
|
-
background.add_task(node.processor.flush_kobj_queue)
|
|
109
|
-
|
|
110
111
|
@app.post(POLL_EVENTS_PATH)
|
|
111
112
|
def poll_events(req: PollEvents) -> EventsPayload:
|
|
112
113
|
logger.info(f"Request to {POLL_EVENTS_PATH}")
|
|
@@ -27,7 +27,9 @@ node = NodeInterface(
|
|
|
27
27
|
profile=NodeProfile(
|
|
28
28
|
node_type=NodeType.PARTIAL
|
|
29
29
|
),
|
|
30
|
-
|
|
30
|
+
cache_directory_path="partial_node_rid_cache",
|
|
31
|
+
event_queues_file_path="parital_node_event_queus.json",
|
|
32
|
+
identity_file_path="partial_node_identity.json",
|
|
31
33
|
first_contact="http://127.0.0.1:8000/koi-net"
|
|
32
34
|
)
|
|
33
35
|
|
|
@@ -72,11 +74,11 @@ def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject):
|
|
|
72
74
|
|
|
73
75
|
|
|
74
76
|
|
|
75
|
-
node.
|
|
77
|
+
node.start()
|
|
76
78
|
|
|
77
79
|
while True:
|
|
78
80
|
for event in node.network.poll_neighbors():
|
|
79
81
|
node.processor.handle(event=event, source=KnowledgeSource.External)
|
|
80
82
|
node.processor.flush_kobj_queue()
|
|
81
83
|
|
|
82
|
-
time.sleep(
|
|
84
|
+
time.sleep(5)
|
|
@@ -2,7 +2,7 @@ import logging
|
|
|
2
2
|
import uvicorn
|
|
3
3
|
from contextlib import asynccontextmanager
|
|
4
4
|
from rich.logging import RichHandler
|
|
5
|
-
from fastapi import FastAPI
|
|
5
|
+
from fastapi import FastAPI
|
|
6
6
|
from rid_lib.types import KoiNetNode, KoiNetEdge
|
|
7
7
|
from koi_net import NodeInterface
|
|
8
8
|
from koi_net.processor.knowledge_object import KnowledgeSource
|
|
@@ -50,27 +50,26 @@ node = NodeInterface(
|
|
|
50
50
|
state=[KoiNetNode, KoiNetEdge]
|
|
51
51
|
)
|
|
52
52
|
),
|
|
53
|
+
use_kobj_processor_thread=True,
|
|
53
54
|
first_contact=coordinator_url
|
|
54
55
|
)
|
|
55
56
|
|
|
56
57
|
|
|
57
58
|
@asynccontextmanager
|
|
58
59
|
async def lifespan(app: FastAPI):
|
|
59
|
-
node.
|
|
60
|
+
node.start()
|
|
60
61
|
yield
|
|
61
|
-
node.
|
|
62
|
+
node.stop()
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
app = FastAPI(lifespan=lifespan, root_path="/koi-net")
|
|
65
66
|
|
|
66
67
|
@app.post(BROADCAST_EVENTS_PATH)
|
|
67
|
-
def broadcast_events(req: EventsPayload
|
|
68
|
+
def broadcast_events(req: EventsPayload):
|
|
68
69
|
logger.info(f"Request to {BROADCAST_EVENTS_PATH}, received {len(req.events)} event(s)")
|
|
69
70
|
for event in req.events:
|
|
70
71
|
node.processor.handle(event=event, source=KnowledgeSource.External)
|
|
71
72
|
|
|
72
|
-
background.add_task(node.processor.flush_kobj_queue)
|
|
73
|
-
|
|
74
73
|
@app.post(POLL_EVENTS_PATH)
|
|
75
74
|
def poll_events(req: PollEvents) -> EventsPayload:
|
|
76
75
|
logger.info(f"Request to {POLL_EVENTS_PATH}")
|
|
@@ -11,25 +11,30 @@ from .protocol.event import Event, EventType
|
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
|
+
|
|
14
15
|
class NodeInterface:
|
|
15
16
|
cache: Cache
|
|
16
17
|
identity: NodeIdentity
|
|
17
18
|
network: NetworkInterface
|
|
18
19
|
processor: ProcessorInterface
|
|
19
20
|
first_contact: str
|
|
21
|
+
use_kobj_processor_thread: bool
|
|
20
22
|
|
|
21
23
|
def __init__(
|
|
22
24
|
self,
|
|
23
25
|
name: str,
|
|
24
26
|
profile: NodeProfile,
|
|
25
27
|
identity_file_path: str = "identity.json",
|
|
28
|
+
event_queues_file_path: str = "event_queues.json",
|
|
29
|
+
cache_directory_path: str = "rid_cache",
|
|
30
|
+
use_kobj_processor_thread: bool = False,
|
|
26
31
|
first_contact: str | None = None,
|
|
27
32
|
handlers: list[KnowledgeHandler] | None = None,
|
|
28
33
|
cache: Cache | None = None,
|
|
29
34
|
network: NetworkInterface | None = None,
|
|
30
35
|
processor: ProcessorInterface | None = None
|
|
31
36
|
):
|
|
32
|
-
self.cache = cache or Cache(
|
|
37
|
+
self.cache = cache or Cache(cache_directory_path)
|
|
33
38
|
self.identity = NodeIdentity(
|
|
34
39
|
name=name,
|
|
35
40
|
profile=profile,
|
|
@@ -38,7 +43,7 @@ class NodeInterface:
|
|
|
38
43
|
)
|
|
39
44
|
self.first_contact = first_contact
|
|
40
45
|
self.network = network or NetworkInterface(
|
|
41
|
-
file_path=
|
|
46
|
+
file_path=event_queues_file_path,
|
|
42
47
|
first_contact=self.first_contact,
|
|
43
48
|
cache=self.cache,
|
|
44
49
|
identity=self.identity
|
|
@@ -50,30 +55,41 @@ class NodeInterface:
|
|
|
50
55
|
obj for obj in vars(default_handlers).values()
|
|
51
56
|
if isinstance(obj, KnowledgeHandler)
|
|
52
57
|
]
|
|
53
|
-
|
|
58
|
+
|
|
59
|
+
self.use_kobj_processor_thread = use_kobj_processor_thread
|
|
54
60
|
self.processor = processor or ProcessorInterface(
|
|
55
61
|
cache=self.cache,
|
|
56
62
|
network=self.network,
|
|
57
63
|
identity=self.identity,
|
|
64
|
+
use_kobj_processor_thread=self.use_kobj_processor_thread,
|
|
58
65
|
default_handlers=handlers
|
|
59
66
|
)
|
|
60
67
|
|
|
61
|
-
def
|
|
62
|
-
"""
|
|
68
|
+
def start(self) -> None:
|
|
69
|
+
"""Starts a node, call this method first.
|
|
63
70
|
|
|
64
|
-
Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
|
|
71
|
+
Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
|
|
65
72
|
"""
|
|
66
|
-
self.
|
|
73
|
+
if self.use_kobj_processor_thread:
|
|
74
|
+
logger.info("Starting processor worker thread")
|
|
75
|
+
self.processor.worker_thread.start()
|
|
67
76
|
|
|
77
|
+
self.network._load_event_queues()
|
|
68
78
|
self.network.graph.generate()
|
|
69
79
|
|
|
70
80
|
self.processor.handle(
|
|
71
81
|
bundle=Bundle.generate(
|
|
72
82
|
rid=self.identity.rid,
|
|
73
83
|
contents=self.identity.profile.model_dump()
|
|
74
|
-
)
|
|
75
|
-
flush=True
|
|
84
|
+
)
|
|
76
85
|
)
|
|
86
|
+
|
|
87
|
+
logger.info("Waiting for kobj queue to empty")
|
|
88
|
+
if self.use_kobj_processor_thread:
|
|
89
|
+
self.processor.kobj_queue.join()
|
|
90
|
+
else:
|
|
91
|
+
self.processor.flush_kobj_queue()
|
|
92
|
+
logger.info("Done")
|
|
77
93
|
|
|
78
94
|
if not self.network.graph.get_neighbors() and self.first_contact:
|
|
79
95
|
logger.info(f"I don't have any neighbors, reaching out to first contact {self.first_contact}")
|
|
@@ -94,9 +110,17 @@ class NodeInterface:
|
|
|
94
110
|
return
|
|
95
111
|
|
|
96
112
|
|
|
97
|
-
def
|
|
98
|
-
"""
|
|
113
|
+
def stop(self):
|
|
114
|
+
"""Stops a node, call this method last.
|
|
99
115
|
|
|
100
|
-
Saves event queues to storage.
|
|
116
|
+
Finishes processing knowledge object queue. Saves event queues to storage.
|
|
101
117
|
"""
|
|
118
|
+
logger.info("Stopping node...")
|
|
119
|
+
|
|
120
|
+
if self.use_kobj_processor_thread:
|
|
121
|
+
logger.info("Waiting for kobj queue to empty")
|
|
122
|
+
self.processor.kobj_queue.join()
|
|
123
|
+
else:
|
|
124
|
+
self.processor.flush_kobj_queue()
|
|
125
|
+
|
|
102
126
|
self.network._save_event_queues()
|
|
@@ -74,13 +74,13 @@ class NetworkGraph:
|
|
|
74
74
|
"""Returns edges this node belongs to.
|
|
75
75
|
|
|
76
76
|
All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
|
|
77
|
-
|
|
77
|
+
|
|
78
78
|
edges = []
|
|
79
|
-
if direction != "in":
|
|
79
|
+
if direction != "in" and self.dg.out_edges:
|
|
80
80
|
out_edges = self.dg.out_edges(self.identity.rid)
|
|
81
81
|
edges.extend([e for e in out_edges])
|
|
82
82
|
|
|
83
|
-
if direction != "out":
|
|
83
|
+
if direction != "out" and self.dg.in_edges:
|
|
84
84
|
in_edges = self.dg.in_edges(self.identity.rid)
|
|
85
85
|
edges.extend([e for e in in_edges])
|
|
86
86
|
|
|
@@ -165,6 +165,8 @@ class NetworkInterface:
|
|
|
165
165
|
return
|
|
166
166
|
|
|
167
167
|
events = self._flush_queue(self.webhook_event_queue, node)
|
|
168
|
+
if not events: return
|
|
169
|
+
|
|
168
170
|
logger.info(f"Broadcasting {len(events)} events")
|
|
169
171
|
|
|
170
172
|
try:
|
|
@@ -173,11 +175,6 @@ class NetworkInterface:
|
|
|
173
175
|
logger.warning("Broadcast failed, requeuing events")
|
|
174
176
|
for event in events:
|
|
175
177
|
self.push_event_to(event, node)
|
|
176
|
-
|
|
177
|
-
def flush_all_webhook_queues(self):
|
|
178
|
-
"""Flushes all nodes' webhook queues and broadcasts events."""
|
|
179
|
-
for node in self.webhook_event_queue.keys():
|
|
180
|
-
self.flush_webhook_queue(node)
|
|
181
178
|
|
|
182
179
|
def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
|
|
183
180
|
"""Returns list of node RIDs which provide state for the specified RID type."""
|
|
@@ -204,8 +201,8 @@ class NetworkInterface:
|
|
|
204
201
|
payload = self.request_handler.fetch_bundles(
|
|
205
202
|
node=node_rid, rids=[rid])
|
|
206
203
|
|
|
207
|
-
if payload.
|
|
208
|
-
remote_bundle = payload.
|
|
204
|
+
if payload.bundles:
|
|
205
|
+
remote_bundle = payload.bundles[0]
|
|
209
206
|
logger.info(f"Got bundle from '{node_rid}'")
|
|
210
207
|
break
|
|
211
208
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
|
|
2
|
+
import queue
|
|
3
|
+
import threading
|
|
3
4
|
from typing import Callable
|
|
4
5
|
from rid_lib.core import RID, RIDType
|
|
5
6
|
from rid_lib.ext import Bundle, Cache, Manifest
|
|
@@ -30,20 +31,30 @@ class ProcessorInterface:
|
|
|
30
31
|
network: NetworkInterface
|
|
31
32
|
identity: NodeIdentity
|
|
32
33
|
handlers: list[KnowledgeHandler]
|
|
33
|
-
kobj_queue: Queue[KnowledgeObject]
|
|
34
|
+
kobj_queue: queue.Queue[KnowledgeObject]
|
|
35
|
+
use_kobj_processor_thread: bool
|
|
36
|
+
worker_thread: threading.Thread | None = None
|
|
34
37
|
|
|
35
38
|
def __init__(
|
|
36
39
|
self,
|
|
37
40
|
cache: Cache,
|
|
38
41
|
network: NetworkInterface,
|
|
39
42
|
identity: NodeIdentity,
|
|
43
|
+
use_kobj_processor_thread: bool,
|
|
40
44
|
default_handlers: list[KnowledgeHandler] = []
|
|
41
45
|
):
|
|
42
46
|
self.cache = cache
|
|
43
47
|
self.network = network
|
|
44
48
|
self.identity = identity
|
|
49
|
+
self.use_kobj_processor_thread = use_kobj_processor_thread
|
|
45
50
|
self.handlers: list[KnowledgeHandler] = default_handlers
|
|
46
|
-
self.kobj_queue = Queue()
|
|
51
|
+
self.kobj_queue = queue.Queue()
|
|
52
|
+
|
|
53
|
+
if self.use_kobj_processor_thread:
|
|
54
|
+
self.worker_thread = threading.Thread(
|
|
55
|
+
target=self.kobj_processor_worker,
|
|
56
|
+
daemon=True
|
|
57
|
+
)
|
|
47
58
|
|
|
48
59
|
def add_handler(self, handler: KnowledgeHandler):
|
|
49
60
|
self.handlers.append(handler)
|
|
@@ -195,18 +206,38 @@ class ProcessorInterface:
|
|
|
195
206
|
logger.info("No network targets set")
|
|
196
207
|
|
|
197
208
|
for node in kobj.network_targets:
|
|
198
|
-
self.network.push_event_to(kobj.normalized_event, node)
|
|
199
|
-
self.network.flush_all_webhook_queues()
|
|
209
|
+
self.network.push_event_to(kobj.normalized_event, node, flush=True)
|
|
200
210
|
|
|
201
211
|
kobj = self.call_handler_chain(HandlerType.Final, kobj)
|
|
202
|
-
|
|
212
|
+
|
|
203
213
|
def flush_kobj_queue(self):
|
|
204
|
-
"""Flushes all knowledge objects from queue and processes them.
|
|
214
|
+
"""Flushes all knowledge objects from queue and processes them.
|
|
215
|
+
|
|
216
|
+
NOTE: ONLY CALL THIS METHOD IN SINGLE THREADED NODES, OTHERWISE THIS WILL CAUSE RACE CONDITIONS.
|
|
217
|
+
"""
|
|
218
|
+
if self.use_kobj_processor_thread:
|
|
219
|
+
logger.warning("You are using a worker thread, calling this method can cause race conditions!")
|
|
220
|
+
|
|
205
221
|
while not self.kobj_queue.empty():
|
|
206
222
|
kobj = self.kobj_queue.get()
|
|
207
223
|
logger.info(f"Dequeued {kobj!r}")
|
|
208
224
|
self.process_kobj(kobj)
|
|
225
|
+
self.kobj_queue.task_done()
|
|
209
226
|
logger.info("Done handling")
|
|
227
|
+
|
|
228
|
+
def kobj_processor_worker(self, timeout=0.1):
|
|
229
|
+
while True:
|
|
230
|
+
try:
|
|
231
|
+
kobj = self.kobj_queue.get(timeout=timeout)
|
|
232
|
+
logger.info(f"Dequeued {kobj!r}")
|
|
233
|
+
self.process_kobj(kobj)
|
|
234
|
+
self.kobj_queue.task_done()
|
|
235
|
+
|
|
236
|
+
except queue.Empty:
|
|
237
|
+
pass
|
|
238
|
+
|
|
239
|
+
except Exception as e:
|
|
240
|
+
logger.warning(f"Error processing kobj: {e}")
|
|
210
241
|
|
|
211
242
|
def handle(
|
|
212
243
|
self,
|
|
@@ -216,8 +247,7 @@ class ProcessorInterface:
|
|
|
216
247
|
event: Event | None = None,
|
|
217
248
|
kobj: KnowledgeObject | None = None,
|
|
218
249
|
event_type: KnowledgeEventType = None,
|
|
219
|
-
source: KnowledgeSource = KnowledgeSource.Internal
|
|
220
|
-
flush: bool = False
|
|
250
|
+
source: KnowledgeSource = KnowledgeSource.Internal
|
|
221
251
|
):
|
|
222
252
|
"""Queues provided knowledge to be handled by processing pipeline.
|
|
223
253
|
|
|
@@ -238,6 +268,3 @@ class ProcessorInterface:
|
|
|
238
268
|
|
|
239
269
|
self.kobj_queue.put(_kobj)
|
|
240
270
|
logger.info(f"Queued {_kobj!r}")
|
|
241
|
-
|
|
242
|
-
if flush:
|
|
243
|
-
self.flush_kobj_queue()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|