koi-net 1.0.0b6__tar.gz → 1.0.0b8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of koi-net might be problematic. Click here for more details.

Files changed (30) hide show
  1. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/PKG-INFO +19 -14
  2. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/README.md +18 -13
  3. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/basic_coordinator_node.py +8 -7
  4. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/basic_partial_node.py +5 -3
  5. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/full_node_template.py +5 -6
  6. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/examples/partial_node_template.py +2 -2
  7. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/pyproject.toml +1 -1
  8. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/core.py +36 -12
  9. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/graph.py +3 -3
  10. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/interface.py +4 -7
  11. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/response_handler.py +1 -1
  12. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/interface.py +39 -12
  13. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/api_models.py +1 -1
  14. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/.gitignore +0 -0
  15. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/LICENSE +0 -0
  16. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/requirements.txt +0 -0
  17. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/__init__.py +0 -0
  18. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/identity.py +0 -0
  19. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/__init__.py +0 -0
  20. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/network/request_handler.py +0 -0
  21. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/__init__.py +0 -0
  22. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/default_handlers.py +0 -0
  23. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/handler.py +0 -0
  24. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/processor/knowledge_object.py +0 -0
  25. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/__init__.py +0 -0
  26. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/consts.py +0 -0
  27. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/edge.py +0 -0
  28. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/event.py +0 -0
  29. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/helpers.py +0 -0
  30. {koi_net-1.0.0b6 → koi_net-1.0.0b8}/src/koi_net/protocol/node.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: koi-net
3
- Version: 1.0.0b6
3
+ Version: 1.0.0b8
4
4
  Summary: Implementation of KOI-net protocol in Python
5
5
  Project-URL: Homepage, https://github.com/BlockScience/koi-net/
6
6
  Author-email: Luke Miller <luke@block.science>
@@ -144,13 +144,14 @@ node = NodeInterface(
144
144
  event=[],
145
145
  state=[]
146
146
  )
147
- )
147
+ ),
148
+ use_kobj_processor_thread=True
148
149
  )
149
150
  ```
150
151
 
151
152
  ## Knowledge Processing
152
153
 
153
- Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.initialize()` and `node.finalize()` at the beginning and end of your node's life cycle.
154
+ Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.start()` and `node.stop()` at the beginning and end of your node's life cycle.
154
155
 
155
156
  ### Partial Node
156
157
  Make sure to set `source=KnowledgeSource.External`, this indicates to the knowledge processing pipeline that the incoming knowledge was received from an external source. Where the knowledge is sourced from will impact decisions in the node's knowledge handlers.
@@ -159,7 +160,7 @@ import time
159
160
  from koi_net.processor.knowledge_object import KnowledgeSource
160
161
 
161
162
  if __name__ == "__main__":
162
- node.initialize()
163
+ node.start()
163
164
 
164
165
  try:
165
166
  while True:
@@ -170,20 +171,20 @@ if __name__ == "__main__":
170
171
  time.sleep(5)
171
172
 
172
173
  finally:
173
- node.finalize()
174
+ node.stop()
174
175
  ```
175
176
 
176
177
  ### Full Node
177
- Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to initialize and finalize the node before and after execution, as well as the FastAPI app which will be our web server.
178
+ Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to start and stop the node before and after execution, as well as the FastAPI app which will be our web server.
178
179
  ```python
179
180
  from contextlib import asynccontextmanager
180
181
  from fastapi import FastAPI
181
182
 
182
183
  @asynccontextmanager
183
184
  async def lifespan(app: FastAPI):
184
- node.initialize()
185
+ node.start()
185
186
  yield
186
- node.finalize()
187
+ node.stop()
187
188
 
188
189
 
189
190
  app = FastAPI(lifespan=lifespan, root_path="/koi-net")
@@ -196,11 +197,9 @@ from koi_net.protocol.api_models import *
196
197
  from koi_net.protocol.consts import *
197
198
 
198
199
  @app.post(BROADCAST_EVENTS_PATH)
199
- def broadcast_events(req: EventsPayload, background: BackgroundTasks):
200
+ def broadcast_events(req: EventsPayload):
200
201
  for event in req.events:
201
202
  node.processor.handle(event=event, source=KnowledgeSource.External)
202
-
203
- background.add_task(node.processor.flush_kobj_queue)
204
203
  ```
205
204
 
206
205
  Next we can add the event polling endpoint, this allows partial nodes to receive events from us.
@@ -265,12 +264,16 @@ class NodeInterface:
265
264
  network: NetworkInterface
266
265
  processor: ProcessorInterface
267
266
  first_contact: str
267
+ use_kobj_processor_thread: bool
268
268
 
269
269
  def __init__(
270
270
  self,
271
271
  name: str,
272
272
  profile: NodeProfile,
273
273
  identity_file_path: str = "identity.json",
274
+ event_queues_file_path: str = "event_queues.json",
275
+ cache_directory_path: str = "rid_cache",
276
+ use_kobj_processor_thread: bool = False,
274
277
  first_contact: str | None = None,
275
278
  handlers: list[KnowledgeHandler] | None = None,
276
279
  cache: Cache | None = None,
@@ -278,8 +281,8 @@ class NodeInterface:
278
281
  processor: ProcessorInterface | None = None
279
282
  ): ...
280
283
 
281
- def initialize(self): ...
282
- def finalize(self): ...
284
+ def start(self): ...
285
+ def stop(self): ...
283
286
  ```
284
287
  As you can see, only a name and profile are required. The other fields allow for additional customization if needed.
285
288
 
@@ -313,7 +316,6 @@ class NetworkInterface:
313
316
 
314
317
  def flush_poll_queue(self, node: KoiNetNode) -> list[Event]: ...
315
318
  def flush_webhook_queue(self, node: RID): ...
316
- def flush_all_webhook_queues(self): ...
317
319
 
318
320
  def fetch_remote_bundle(self, rid: RID): ...
319
321
  def fetch_remote_manifest(self, rid: RID): ...
@@ -432,11 +434,14 @@ def poll_events(req: PollEvents) -> EventsPayload:
432
434
  The `ProcessorInterface` class provides access to a node's internal knowledge processing pipeline.
433
435
  ```python
434
436
  class ProcessorInterface:
437
+ worker_thread: threading.Thread | None = None
438
+
435
439
  def __init__(
436
440
  self,
437
441
  cache: Cache,
438
442
  network: NetworkInterface,
439
443
  identity: NodeIdentity,
444
+ use_kobj_processor_thread: bool,
440
445
  default_handlers: list[KnowledgeHandler] = []
441
446
  ): ...
442
447
 
@@ -102,13 +102,14 @@ node = NodeInterface(
102
102
  event=[],
103
103
  state=[]
104
104
  )
105
- )
105
+ ),
106
+ use_kobj_processor_thread=True
106
107
  )
107
108
  ```
108
109
 
109
110
  ## Knowledge Processing
110
111
 
111
- Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.initialize()` and `node.finalize()` at the beginning and end of your node's life cycle.
112
+ Next we'll set up the knowledge processing flow for our node. This is where most of the node's logic and behavior will come into play. For partial nodes this will be an event loop, and for full nodes we will use webhooks. Make sure to call `node.start()` and `node.stop()` at the beginning and end of your node's life cycle.
112
113
 
113
114
  ### Partial Node
114
115
  Make sure to set `source=KnowledgeSource.External`, this indicates to the knowledge processing pipeline that the incoming knowledge was received from an external source. Where the knowledge is sourced from will impact decisions in the node's knowledge handlers.
@@ -117,7 +118,7 @@ import time
117
118
  from koi_net.processor.knowledge_object import KnowledgeSource
118
119
 
119
120
  if __name__ == "__main__":
120
- node.initialize()
121
+ node.start()
121
122
 
122
123
  try:
123
124
  while True:
@@ -128,20 +129,20 @@ if __name__ == "__main__":
128
129
  time.sleep(5)
129
130
 
130
131
  finally:
131
- node.finalize()
132
+ node.stop()
132
133
  ```
133
134
 
134
135
  ### Full Node
135
- Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to initialize and finalize the node before and after execution, as well as the FastAPI app which will be our web server.
136
+ Setting up a full node is slightly more complex as we'll need a webserver. For this example, we'll use FastAPI and uvicorn. First we need to setup the "lifespan" of the server, to start and stop the node before and after execution, as well as the FastAPI app which will be our web server.
136
137
  ```python
137
138
  from contextlib import asynccontextmanager
138
139
  from fastapi import FastAPI
139
140
 
140
141
  @asynccontextmanager
141
142
  async def lifespan(app: FastAPI):
142
- node.initialize()
143
+ node.start()
143
144
  yield
144
- node.finalize()
145
+ node.stop()
145
146
 
146
147
 
147
148
  app = FastAPI(lifespan=lifespan, root_path="/koi-net")
@@ -154,11 +155,9 @@ from koi_net.protocol.api_models import *
154
155
  from koi_net.protocol.consts import *
155
156
 
156
157
  @app.post(BROADCAST_EVENTS_PATH)
157
- def broadcast_events(req: EventsPayload, background: BackgroundTasks):
158
+ def broadcast_events(req: EventsPayload):
158
159
  for event in req.events:
159
160
  node.processor.handle(event=event, source=KnowledgeSource.External)
160
-
161
- background.add_task(node.processor.flush_kobj_queue)
162
161
  ```
163
162
 
164
163
  Next we can add the event polling endpoint, this allows partial nodes to receive events from us.
@@ -223,12 +222,16 @@ class NodeInterface:
223
222
  network: NetworkInterface
224
223
  processor: ProcessorInterface
225
224
  first_contact: str
225
+ use_kobj_processor_thread: bool
226
226
 
227
227
  def __init__(
228
228
  self,
229
229
  name: str,
230
230
  profile: NodeProfile,
231
231
  identity_file_path: str = "identity.json",
232
+ event_queues_file_path: str = "event_queues.json",
233
+ cache_directory_path: str = "rid_cache",
234
+ use_kobj_processor_thread: bool = False,
232
235
  first_contact: str | None = None,
233
236
  handlers: list[KnowledgeHandler] | None = None,
234
237
  cache: Cache | None = None,
@@ -236,8 +239,8 @@ class NodeInterface:
236
239
  processor: ProcessorInterface | None = None
237
240
  ): ...
238
241
 
239
- def initialize(self): ...
240
- def finalize(self): ...
242
+ def start(self): ...
243
+ def stop(self): ...
241
244
  ```
242
245
  As you can see, only a name and profile are required. The other fields allow for additional customization if needed.
243
246
 
@@ -271,7 +274,6 @@ class NetworkInterface:
271
274
 
272
275
  def flush_poll_queue(self, node: KoiNetNode) -> list[Event]: ...
273
276
  def flush_webhook_queue(self, node: RID): ...
274
- def flush_all_webhook_queues(self): ...
275
277
 
276
278
  def fetch_remote_bundle(self, rid: RID): ...
277
279
  def fetch_remote_manifest(self, rid: RID): ...
@@ -390,11 +392,14 @@ def poll_events(req: PollEvents) -> EventsPayload:
390
392
  The `ProcessorInterface` class provides access to a node's internal knowledge processing pipeline.
391
393
  ```python
392
394
  class ProcessorInterface:
395
+ worker_thread: threading.Thread | None = None
396
+
393
397
  def __init__(
394
398
  self,
395
399
  cache: Cache,
396
400
  network: NetworkInterface,
397
401
  identity: NodeIdentity,
402
+ use_kobj_processor_thread: bool,
398
403
  default_handlers: list[KnowledgeHandler] = []
399
404
  ): ...
400
405
 
@@ -3,7 +3,7 @@ import logging
3
3
  import uvicorn
4
4
  from contextlib import asynccontextmanager
5
5
  from rich.logging import RichHandler
6
- from fastapi import FastAPI, BackgroundTasks
6
+ from fastapi import FastAPI
7
7
  from rid_lib.types import KoiNetNode, KoiNetEdge
8
8
  from koi_net import NodeInterface
9
9
  from koi_net.processor.handler import HandlerType
@@ -53,7 +53,10 @@ node = NodeInterface(
53
53
  state=[KoiNetNode, KoiNetEdge]
54
54
  )
55
55
  ),
56
- identity_file_path="coordinator_identity.json"
56
+ use_kobj_processor_thread=True,
57
+ cache_directory_path="coordinator_node_rid_cache",
58
+ event_queues_file_path="coordinator_node_event_queus.json",
59
+ identity_file_path="coordinator_node_identity.json",
57
60
  )
58
61
 
59
62
 
@@ -88,9 +91,9 @@ def handshake_handler(proc: ProcessorInterface, kobj: KnowledgeObject):
88
91
 
89
92
  @asynccontextmanager
90
93
  async def lifespan(app: FastAPI):
91
- node.initialize()
94
+ node.start()
92
95
  yield
93
- node.finalize()
96
+ node.stop()
94
97
 
95
98
  app = FastAPI(
96
99
  lifespan=lifespan,
@@ -100,13 +103,11 @@ app = FastAPI(
100
103
  )
101
104
 
102
105
  @app.post(BROADCAST_EVENTS_PATH)
103
- def broadcast_events(req: EventsPayload, background: BackgroundTasks):
106
+ def broadcast_events(req: EventsPayload):
104
107
  logger.info(f"Request to {BROADCAST_EVENTS_PATH}, received {len(req.events)} event(s)")
105
108
  for event in req.events:
106
109
  node.processor.handle(event=event, source=KnowledgeSource.External)
107
110
 
108
- background.add_task(node.processor.flush_kobj_queue)
109
-
110
111
  @app.post(POLL_EVENTS_PATH)
111
112
  def poll_events(req: PollEvents) -> EventsPayload:
112
113
  logger.info(f"Request to {POLL_EVENTS_PATH}")
@@ -27,7 +27,9 @@ node = NodeInterface(
27
27
  profile=NodeProfile(
28
28
  node_type=NodeType.PARTIAL
29
29
  ),
30
- identity_file_path="partial_identity.json",
30
+ cache_directory_path="partial_node_rid_cache",
31
+ event_queues_file_path="parital_node_event_queus.json",
32
+ identity_file_path="partial_node_identity.json",
31
33
  first_contact="http://127.0.0.1:8000/koi-net"
32
34
  )
33
35
 
@@ -72,11 +74,11 @@ def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject):
72
74
 
73
75
 
74
76
 
75
- node.initialize()
77
+ node.start()
76
78
 
77
79
  while True:
78
80
  for event in node.network.poll_neighbors():
79
81
  node.processor.handle(event=event, source=KnowledgeSource.External)
80
82
  node.processor.flush_kobj_queue()
81
83
 
82
- time.sleep(1)
84
+ time.sleep(5)
@@ -2,7 +2,7 @@ import logging
2
2
  import uvicorn
3
3
  from contextlib import asynccontextmanager
4
4
  from rich.logging import RichHandler
5
- from fastapi import FastAPI, BackgroundTasks
5
+ from fastapi import FastAPI
6
6
  from rid_lib.types import KoiNetNode, KoiNetEdge
7
7
  from koi_net import NodeInterface
8
8
  from koi_net.processor.knowledge_object import KnowledgeSource
@@ -50,27 +50,26 @@ node = NodeInterface(
50
50
  state=[KoiNetNode, KoiNetEdge]
51
51
  )
52
52
  ),
53
+ use_kobj_processor_thread=True,
53
54
  first_contact=coordinator_url
54
55
  )
55
56
 
56
57
 
57
58
  @asynccontextmanager
58
59
  async def lifespan(app: FastAPI):
59
- node.initialize()
60
+ node.start()
60
61
  yield
61
- node.finalize()
62
+ node.stop()
62
63
 
63
64
 
64
65
  app = FastAPI(lifespan=lifespan, root_path="/koi-net")
65
66
 
66
67
  @app.post(BROADCAST_EVENTS_PATH)
67
- def broadcast_events(req: EventsPayload, background: BackgroundTasks):
68
+ def broadcast_events(req: EventsPayload):
68
69
  logger.info(f"Request to {BROADCAST_EVENTS_PATH}, received {len(req.events)} event(s)")
69
70
  for event in req.events:
70
71
  node.processor.handle(event=event, source=KnowledgeSource.External)
71
72
 
72
- background.add_task(node.processor.flush_kobj_queue)
73
-
74
73
  @app.post(POLL_EVENTS_PATH)
75
74
  def poll_events(req: PollEvents) -> EventsPayload:
76
75
  logger.info(f"Request to {POLL_EVENTS_PATH}")
@@ -27,7 +27,7 @@ node = NodeInterface(
27
27
  )
28
28
 
29
29
  if __name__ == "__main__":
30
- node.initialize()
30
+ node.start()
31
31
 
32
32
  try:
33
33
  while True:
@@ -38,4 +38,4 @@ if __name__ == "__main__":
38
38
  time.sleep(5)
39
39
 
40
40
  finally:
41
- node.finalize()
41
+ node.stop()
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "koi-net"
7
- version = "1.0.0-beta.6"
7
+ version = "1.0.0-beta.8"
8
8
  description = "Implementation of KOI-net protocol in Python"
9
9
  authors = [
10
10
  {name = "Luke Miller", email = "luke@block.science"}
@@ -11,25 +11,30 @@ from .protocol.event import Event, EventType
11
11
 
12
12
  logger = logging.getLogger(__name__)
13
13
 
14
+
14
15
  class NodeInterface:
15
16
  cache: Cache
16
17
  identity: NodeIdentity
17
18
  network: NetworkInterface
18
19
  processor: ProcessorInterface
19
20
  first_contact: str
21
+ use_kobj_processor_thread: bool
20
22
 
21
23
  def __init__(
22
24
  self,
23
25
  name: str,
24
26
  profile: NodeProfile,
25
27
  identity_file_path: str = "identity.json",
28
+ event_queues_file_path: str = "event_queues.json",
29
+ cache_directory_path: str = "rid_cache",
30
+ use_kobj_processor_thread: bool = False,
26
31
  first_contact: str | None = None,
27
32
  handlers: list[KnowledgeHandler] | None = None,
28
33
  cache: Cache | None = None,
29
34
  network: NetworkInterface | None = None,
30
35
  processor: ProcessorInterface | None = None
31
36
  ):
32
- self.cache = cache or Cache(directory_path=f".cache")
37
+ self.cache = cache or Cache(cache_directory_path)
33
38
  self.identity = NodeIdentity(
34
39
  name=name,
35
40
  profile=profile,
@@ -38,7 +43,7 @@ class NodeInterface:
38
43
  )
39
44
  self.first_contact = first_contact
40
45
  self.network = network or NetworkInterface(
41
- file_path="event_queues.json",
46
+ file_path=event_queues_file_path,
42
47
  first_contact=self.first_contact,
43
48
  cache=self.cache,
44
49
  identity=self.identity
@@ -50,30 +55,41 @@ class NodeInterface:
50
55
  obj for obj in vars(default_handlers).values()
51
56
  if isinstance(obj, KnowledgeHandler)
52
57
  ]
53
-
58
+
59
+ self.use_kobj_processor_thread = use_kobj_processor_thread
54
60
  self.processor = processor or ProcessorInterface(
55
61
  cache=self.cache,
56
62
  network=self.network,
57
63
  identity=self.identity,
64
+ use_kobj_processor_thread=self.use_kobj_processor_thread,
58
65
  default_handlers=handlers
59
66
  )
60
67
 
61
- def initialize(self) -> None:
62
- """Initializes node, call on startup.
68
+ def start(self) -> None:
69
+ """Starts a node, call this method first.
63
70
 
64
- Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
71
+ Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
65
72
  """
66
- self.network._load_event_queues()
73
+ if self.use_kobj_processor_thread:
74
+ logger.info("Starting processor worker thread")
75
+ self.processor.worker_thread.start()
67
76
 
77
+ self.network._load_event_queues()
68
78
  self.network.graph.generate()
69
79
 
70
80
  self.processor.handle(
71
81
  bundle=Bundle.generate(
72
82
  rid=self.identity.rid,
73
83
  contents=self.identity.profile.model_dump()
74
- ),
75
- flush=True
84
+ )
76
85
  )
86
+
87
+ logger.info("Waiting for kobj queue to empty")
88
+ if self.use_kobj_processor_thread:
89
+ self.processor.kobj_queue.join()
90
+ else:
91
+ self.processor.flush_kobj_queue()
92
+ logger.info("Done")
77
93
 
78
94
  if not self.network.graph.get_neighbors() and self.first_contact:
79
95
  logger.info(f"I don't have any neighbors, reaching out to first contact {self.first_contact}")
@@ -94,9 +110,17 @@ class NodeInterface:
94
110
  return
95
111
 
96
112
 
97
- def finalize(self):
98
- """Finalizes node, call on shutdown.
113
+ def stop(self):
114
+ """Stops a node, call this method last.
99
115
 
100
- Saves event queues to storage.
116
+ Finishes processing knowledge object queue. Saves event queues to storage.
101
117
  """
118
+ logger.info("Stopping node...")
119
+
120
+ if self.use_kobj_processor_thread:
121
+ logger.info("Waiting for kobj queue to empty")
122
+ self.processor.kobj_queue.join()
123
+ else:
124
+ self.processor.flush_kobj_queue()
125
+
102
126
  self.network._save_event_queues()
@@ -74,13 +74,13 @@ class NetworkGraph:
74
74
  """Returns edges this node belongs to.
75
75
 
76
76
  All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
77
-
77
+
78
78
  edges = []
79
- if direction != "in":
79
+ if direction != "in" and self.dg.out_edges:
80
80
  out_edges = self.dg.out_edges(self.identity.rid)
81
81
  edges.extend([e for e in out_edges])
82
82
 
83
- if direction != "out":
83
+ if direction != "out" and self.dg.in_edges:
84
84
  in_edges = self.dg.in_edges(self.identity.rid)
85
85
  edges.extend([e for e in in_edges])
86
86
 
@@ -165,6 +165,8 @@ class NetworkInterface:
165
165
  return
166
166
 
167
167
  events = self._flush_queue(self.webhook_event_queue, node)
168
+ if not events: return
169
+
168
170
  logger.info(f"Broadcasting {len(events)} events")
169
171
 
170
172
  try:
@@ -173,11 +175,6 @@ class NetworkInterface:
173
175
  logger.warning("Broadcast failed, requeuing events")
174
176
  for event in events:
175
177
  self.push_event_to(event, node)
176
-
177
- def flush_all_webhook_queues(self):
178
- """Flushes all nodes' webhook queues and broadcasts events."""
179
- for node in self.webhook_event_queue.keys():
180
- self.flush_webhook_queue(node)
181
178
 
182
179
  def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
183
180
  """Returns list of node RIDs which provide state for the specified RID type."""
@@ -204,8 +201,8 @@ class NetworkInterface:
204
201
  payload = self.request_handler.fetch_bundles(
205
202
  node=node_rid, rids=[rid])
206
203
 
207
- if payload.manifests:
208
- remote_bundle = payload.manifests[0]
204
+ if payload.bundles:
205
+ remote_bundle = payload.bundles[0]
209
206
  logger.info(f"Got bundle from '{node_rid}'")
210
207
  break
211
208
 
@@ -56,4 +56,4 @@ class ResponseHandler:
56
56
  else:
57
57
  not_found.append(rid)
58
58
 
59
- return BundlesPayload(manifests=bundles, not_found=not_found)
59
+ return BundlesPayload(bundles=bundles, not_found=not_found)
@@ -1,5 +1,6 @@
1
1
  import logging
2
- from queue import Queue
2
+ import queue
3
+ import threading
3
4
  from typing import Callable
4
5
  from rid_lib.core import RID, RIDType
5
6
  from rid_lib.ext import Bundle, Cache, Manifest
@@ -30,20 +31,30 @@ class ProcessorInterface:
30
31
  network: NetworkInterface
31
32
  identity: NodeIdentity
32
33
  handlers: list[KnowledgeHandler]
33
- kobj_queue: Queue[KnowledgeObject]
34
+ kobj_queue: queue.Queue[KnowledgeObject]
35
+ use_kobj_processor_thread: bool
36
+ worker_thread: threading.Thread | None = None
34
37
 
35
38
  def __init__(
36
39
  self,
37
40
  cache: Cache,
38
41
  network: NetworkInterface,
39
42
  identity: NodeIdentity,
43
+ use_kobj_processor_thread: bool,
40
44
  default_handlers: list[KnowledgeHandler] = []
41
45
  ):
42
46
  self.cache = cache
43
47
  self.network = network
44
48
  self.identity = identity
49
+ self.use_kobj_processor_thread = use_kobj_processor_thread
45
50
  self.handlers: list[KnowledgeHandler] = default_handlers
46
- self.kobj_queue = Queue()
51
+ self.kobj_queue = queue.Queue()
52
+
53
+ if self.use_kobj_processor_thread:
54
+ self.worker_thread = threading.Thread(
55
+ target=self.kobj_processor_worker,
56
+ daemon=True
57
+ )
47
58
 
48
59
  def add_handler(self, handler: KnowledgeHandler):
49
60
  self.handlers.append(handler)
@@ -195,18 +206,38 @@ class ProcessorInterface:
195
206
  logger.info("No network targets set")
196
207
 
197
208
  for node in kobj.network_targets:
198
- self.network.push_event_to(kobj.normalized_event, node)
199
- self.network.flush_all_webhook_queues()
209
+ self.network.push_event_to(kobj.normalized_event, node, flush=True)
200
210
 
201
211
  kobj = self.call_handler_chain(HandlerType.Final, kobj)
202
-
212
+
203
213
  def flush_kobj_queue(self):
204
- """Flushes all knowledge objects from queue and processes them."""
214
+ """Flushes all knowledge objects from queue and processes them.
215
+
216
+ NOTE: ONLY CALL THIS METHOD IN SINGLE THREADED NODES, OTHERWISE THIS WILL CAUSE RACE CONDITIONS.
217
+ """
218
+ if self.use_kobj_processor_thread:
219
+ logger.warning("You are using a worker thread, calling this method can cause race conditions!")
220
+
205
221
  while not self.kobj_queue.empty():
206
222
  kobj = self.kobj_queue.get()
207
223
  logger.info(f"Dequeued {kobj!r}")
208
224
  self.process_kobj(kobj)
225
+ self.kobj_queue.task_done()
209
226
  logger.info("Done handling")
227
+
228
+ def kobj_processor_worker(self, timeout=0.1):
229
+ while True:
230
+ try:
231
+ kobj = self.kobj_queue.get(timeout=timeout)
232
+ logger.info(f"Dequeued {kobj!r}")
233
+ self.process_kobj(kobj)
234
+ self.kobj_queue.task_done()
235
+
236
+ except queue.Empty:
237
+ pass
238
+
239
+ except Exception as e:
240
+ logger.warning(f"Error processing kobj: {e}")
210
241
 
211
242
  def handle(
212
243
  self,
@@ -216,8 +247,7 @@ class ProcessorInterface:
216
247
  event: Event | None = None,
217
248
  kobj: KnowledgeObject | None = None,
218
249
  event_type: KnowledgeEventType = None,
219
- source: KnowledgeSource = KnowledgeSource.Internal,
220
- flush: bool = False
250
+ source: KnowledgeSource = KnowledgeSource.Internal
221
251
  ):
222
252
  """Queues provided knowledge to be handled by processing pipeline.
223
253
 
@@ -238,6 +268,3 @@ class ProcessorInterface:
238
268
 
239
269
  self.kobj_queue.put(_kobj)
240
270
  logger.info(f"Queued {_kobj!r}")
241
-
242
- if flush:
243
- self.flush_kobj_queue()
@@ -33,7 +33,7 @@ class ManifestsPayload(BaseModel):
33
33
  not_found: list[RID] = []
34
34
 
35
35
  class BundlesPayload(BaseModel):
36
- manifests: list[Bundle]
36
+ bundles: list[Bundle]
37
37
  not_found: list[RID] = []
38
38
  deferred: list[RID] = []
39
39
 
File without changes
File without changes
File without changes