koi-net 1.2.0b1__py3-none-any.whl → 1.2.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- koi_net/__init__.py +1 -1
- koi_net/assembler.py +95 -0
- koi_net/cli/models.py +2 -2
- koi_net/config/core.py +71 -0
- koi_net/config/full_node.py +31 -0
- koi_net/config/loader.py +46 -0
- koi_net/config/partial_node.py +18 -0
- koi_net/core.py +43 -206
- koi_net/default_actions.py +1 -2
- koi_net/effector.py +27 -15
- koi_net/entrypoints/__init__.py +2 -0
- koi_net/entrypoints/base.py +5 -0
- koi_net/{poller.py → entrypoints/poller.py} +14 -12
- koi_net/entrypoints/server.py +94 -0
- koi_net/handshaker.py +5 -5
- koi_net/identity.py +3 -4
- koi_net/lifecycle.py +42 -34
- koi_net/logger.py +176 -0
- koi_net/network/error_handler.py +7 -7
- koi_net/network/event_queue.py +9 -7
- koi_net/network/graph.py +8 -8
- koi_net/network/poll_event_buffer.py +26 -0
- koi_net/network/request_handler.py +23 -28
- koi_net/network/resolver.py +14 -14
- koi_net/network/response_handler.py +74 -9
- koi_net/{context.py → processor/context.py} +11 -19
- koi_net/processor/handler.py +4 -1
- koi_net/processor/{default_handlers.py → knowledge_handlers.py} +32 -31
- koi_net/processor/knowledge_object.py +2 -3
- koi_net/processor/kobj_queue.py +4 -4
- koi_net/processor/{knowledge_pipeline.py → pipeline.py} +25 -28
- koi_net/protocol/api_models.py +5 -2
- koi_net/protocol/envelope.py +5 -6
- koi_net/protocol/model_map.py +61 -0
- koi_net/protocol/node.py +3 -3
- koi_net/protocol/secure.py +14 -8
- koi_net/secure.py +6 -7
- koi_net/workers/__init__.py +2 -0
- koi_net/{worker.py → workers/base.py} +7 -0
- koi_net/{processor → workers}/event_worker.py +19 -23
- koi_net/{kobj_worker.py → workers/kobj_worker.py} +12 -13
- {koi_net-1.2.0b1.dist-info → koi_net-1.2.0b3.dist-info}/METADATA +2 -1
- koi_net-1.2.0b3.dist-info/RECORD +56 -0
- koi_net/behaviors.py +0 -51
- koi_net/config.py +0 -161
- koi_net/models.py +0 -14
- koi_net/poll_event_buffer.py +0 -17
- koi_net/server.py +0 -145
- koi_net-1.2.0b1.dist-info/RECORD +0 -49
- {koi_net-1.2.0b1.dist-info → koi_net-1.2.0b3.dist-info}/WHEEL +0 -0
- {koi_net-1.2.0b1.dist-info → koi_net-1.2.0b3.dist-info}/entry_points.txt +0 -0
- {koi_net-1.2.0b1.dist-info → koi_net-1.2.0b3.dist-info}/licenses/LICENSE +0 -0
koi_net/processor/kobj_queue.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import structlog
|
|
2
2
|
from queue import Queue
|
|
3
3
|
from rid_lib.core import RID
|
|
4
4
|
from rid_lib.ext import Bundle, Manifest
|
|
@@ -6,7 +6,7 @@ from rid_lib.types import KoiNetNode
|
|
|
6
6
|
from ..protocol.event import Event, EventType
|
|
7
7
|
from .knowledge_object import KnowledgeObject
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
log = structlog.stdlib.get_logger()
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class KobjQueue:
|
|
@@ -16,7 +16,7 @@ class KobjQueue:
|
|
|
16
16
|
def __init__(self):
|
|
17
17
|
self.q = Queue()
|
|
18
18
|
|
|
19
|
-
def
|
|
19
|
+
def push(
|
|
20
20
|
self,
|
|
21
21
|
rid: RID | None = None,
|
|
22
22
|
manifest: Manifest | None = None,
|
|
@@ -48,4 +48,4 @@ class KobjQueue:
|
|
|
48
48
|
raise ValueError("One of 'rid', 'manifest', 'bundle', 'event', or 'kobj' must be provided")
|
|
49
49
|
|
|
50
50
|
self.q.put(_kobj)
|
|
51
|
-
|
|
51
|
+
log.debug(f"Queued {_kobj!r}")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import structlog
|
|
2
2
|
from rid_lib.types import KoiNetEdge, KoiNetNode
|
|
3
3
|
from rid_lib.ext import Cache
|
|
4
4
|
from ..protocol.event import EventType
|
|
@@ -13,16 +13,13 @@ from .handler import (
|
|
|
13
13
|
StopChain
|
|
14
14
|
)
|
|
15
15
|
from .knowledge_object import KnowledgeObject
|
|
16
|
+
from .context import HandlerContext
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from ..context import HandlerContext
|
|
20
|
-
|
|
21
|
-
logger = logging.getLogger(__name__)
|
|
18
|
+
log = structlog.stdlib.get_logger()
|
|
22
19
|
|
|
23
20
|
|
|
24
21
|
class KnowledgePipeline:
|
|
25
|
-
handler_context:
|
|
22
|
+
handler_context: HandlerContext
|
|
26
23
|
cache: Cache
|
|
27
24
|
identity: NodeIdentity
|
|
28
25
|
request_handler: RequestHandler
|
|
@@ -32,7 +29,7 @@ class KnowledgePipeline:
|
|
|
32
29
|
|
|
33
30
|
def __init__(
|
|
34
31
|
self,
|
|
35
|
-
handler_context:
|
|
32
|
+
handler_context: HandlerContext,
|
|
36
33
|
cache: Cache,
|
|
37
34
|
request_handler: RequestHandler,
|
|
38
35
|
event_queue: EventQueue,
|
|
@@ -71,7 +68,7 @@ class KnowledgePipeline:
|
|
|
71
68
|
if handler.event_types and kobj.event_type not in handler.event_types:
|
|
72
69
|
continue
|
|
73
70
|
|
|
74
|
-
|
|
71
|
+
log.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
|
|
75
72
|
|
|
76
73
|
resp = handler.func(
|
|
77
74
|
ctx=self.handler_context,
|
|
@@ -80,7 +77,7 @@ class KnowledgePipeline:
|
|
|
80
77
|
|
|
81
78
|
# stops handler chain execution
|
|
82
79
|
if resp is STOP_CHAIN:
|
|
83
|
-
|
|
80
|
+
log.debug(f"Handler chain stopped by {handler.func.__name__}")
|
|
84
81
|
return STOP_CHAIN
|
|
85
82
|
# kobj unmodified
|
|
86
83
|
elif resp is None:
|
|
@@ -88,7 +85,7 @@ class KnowledgePipeline:
|
|
|
88
85
|
# kobj modified by handler
|
|
89
86
|
elif isinstance(resp, KnowledgeObject):
|
|
90
87
|
kobj = resp
|
|
91
|
-
|
|
88
|
+
log.debug(f"Knowledge object modified by {handler.func.__name__}")
|
|
92
89
|
else:
|
|
93
90
|
raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
|
|
94
91
|
|
|
@@ -107,36 +104,36 @@ class KnowledgePipeline:
|
|
|
107
104
|
The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
|
|
108
105
|
"""
|
|
109
106
|
|
|
110
|
-
|
|
107
|
+
log.debug(f"Handling {kobj!r}")
|
|
111
108
|
kobj = self.call_handler_chain(HandlerType.RID, kobj)
|
|
112
109
|
if kobj is STOP_CHAIN: return
|
|
113
110
|
|
|
114
111
|
if kobj.event_type == EventType.FORGET:
|
|
115
112
|
bundle = self.cache.read(kobj.rid)
|
|
116
113
|
if not bundle:
|
|
117
|
-
|
|
114
|
+
log.debug("Local bundle not found")
|
|
118
115
|
return
|
|
119
116
|
|
|
120
117
|
# the bundle (to be deleted) attached to kobj for downstream analysis
|
|
121
|
-
|
|
118
|
+
log.debug("Adding local bundle (to be deleted) to knowledge object")
|
|
122
119
|
kobj.manifest = bundle.manifest
|
|
123
120
|
kobj.contents = bundle.contents
|
|
124
121
|
|
|
125
122
|
else:
|
|
126
123
|
# attempt to retrieve manifest
|
|
127
124
|
if not kobj.manifest:
|
|
128
|
-
|
|
125
|
+
log.debug("Manifest not found")
|
|
129
126
|
if not kobj.source:
|
|
130
127
|
return
|
|
131
128
|
|
|
132
|
-
|
|
129
|
+
log.debug("Attempting to fetch remote manifest from source")
|
|
133
130
|
payload = self.request_handler.fetch_manifests(
|
|
134
131
|
node=kobj.source,
|
|
135
132
|
rids=[kobj.rid]
|
|
136
133
|
)
|
|
137
134
|
|
|
138
135
|
if not payload.manifests:
|
|
139
|
-
|
|
136
|
+
log.debug("Failed to find manifest")
|
|
140
137
|
return
|
|
141
138
|
|
|
142
139
|
kobj.manifest = payload.manifests[0]
|
|
@@ -146,24 +143,24 @@ class KnowledgePipeline:
|
|
|
146
143
|
|
|
147
144
|
# attempt to retrieve bundle
|
|
148
145
|
if not kobj.bundle:
|
|
149
|
-
|
|
146
|
+
log.debug("Bundle not found")
|
|
150
147
|
if kobj.source is None:
|
|
151
148
|
return
|
|
152
149
|
|
|
153
|
-
|
|
150
|
+
log.debug("Attempting to fetch remote bundle from source")
|
|
154
151
|
payload = self.request_handler.fetch_bundles(
|
|
155
152
|
node=kobj.source,
|
|
156
153
|
rids=[kobj.rid]
|
|
157
154
|
)
|
|
158
155
|
|
|
159
156
|
if not payload.bundles:
|
|
160
|
-
|
|
157
|
+
log.debug("Failed to find bundle")
|
|
161
158
|
return
|
|
162
159
|
|
|
163
160
|
bundle = payload.bundles[0]
|
|
164
161
|
|
|
165
162
|
if kobj.manifest != bundle.manifest:
|
|
166
|
-
|
|
163
|
+
log.warning("Retrieved bundle contains a different manifest")
|
|
167
164
|
|
|
168
165
|
kobj.manifest = bundle.manifest
|
|
169
166
|
kobj.contents = bundle.contents
|
|
@@ -172,30 +169,30 @@ class KnowledgePipeline:
|
|
|
172
169
|
if kobj is STOP_CHAIN: return
|
|
173
170
|
|
|
174
171
|
if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
|
|
175
|
-
|
|
172
|
+
log.info(f"Writing to cache: {kobj!r}")
|
|
176
173
|
self.cache.write(kobj.bundle)
|
|
177
174
|
|
|
178
175
|
elif kobj.normalized_event_type == EventType.FORGET:
|
|
179
|
-
|
|
176
|
+
log.info(f"Deleting from cache: {kobj!r}")
|
|
180
177
|
self.cache.delete(kobj.rid)
|
|
181
178
|
|
|
182
179
|
else:
|
|
183
|
-
|
|
180
|
+
log.debug("Normalized event type was never set, no cache or network operations will occur")
|
|
184
181
|
return
|
|
185
182
|
|
|
186
183
|
if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
|
|
187
|
-
|
|
184
|
+
log.debug("Change to node or edge, regenerating network graph")
|
|
188
185
|
self.graph.generate()
|
|
189
186
|
|
|
190
187
|
kobj = self.call_handler_chain(HandlerType.Network, kobj)
|
|
191
188
|
if kobj is STOP_CHAIN: return
|
|
192
189
|
|
|
193
190
|
if kobj.network_targets:
|
|
194
|
-
|
|
191
|
+
log.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
|
|
195
192
|
else:
|
|
196
|
-
|
|
193
|
+
log.debug("No network targets set")
|
|
197
194
|
|
|
198
195
|
for node in kobj.network_targets:
|
|
199
|
-
self.event_queue.
|
|
196
|
+
self.event_queue.push(kobj.normalized_event, node)
|
|
200
197
|
|
|
201
198
|
kobj = self.call_handler_chain(HandlerType.Final, kobj)
|
koi_net/protocol/api_models.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Pydantic models for request and response objects in the KOI-net API."""
|
|
2
2
|
|
|
3
|
-
from typing import Literal
|
|
3
|
+
from typing import Annotated, Literal
|
|
4
4
|
from pydantic import BaseModel, Field
|
|
5
5
|
from rid_lib import RID, RIDType
|
|
6
6
|
from rid_lib.ext import Bundle, Manifest
|
|
@@ -60,4 +60,7 @@ class ErrorResponse(BaseModel):
|
|
|
60
60
|
|
|
61
61
|
type RequestModels = EventsPayload | PollEvents | FetchRids | FetchManifests | FetchBundles
|
|
62
62
|
type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload | ErrorResponse
|
|
63
|
-
type ApiModels =
|
|
63
|
+
type ApiModels = Annotated[
|
|
64
|
+
RequestModels | ResponseModels,
|
|
65
|
+
Field(discriminator="type")
|
|
66
|
+
]
|
koi_net/protocol/envelope.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import structlog
|
|
2
2
|
from typing import Generic, TypeVar
|
|
3
3
|
from pydantic import BaseModel, ConfigDict
|
|
4
4
|
from rid_lib.types import KoiNetNode
|
|
@@ -6,8 +6,7 @@ from rid_lib.types import KoiNetNode
|
|
|
6
6
|
from .secure import PrivateKey, PublicKey
|
|
7
7
|
from .api_models import RequestModels, ResponseModels
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
logger = logging.getLogger(__name__)
|
|
9
|
+
log = structlog.stdlib.get_logger()
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
T = TypeVar("T", bound=RequestModels | ResponseModels)
|
|
@@ -28,7 +27,7 @@ class SignedEnvelope(BaseModel, Generic[T]):
|
|
|
28
27
|
target_node=self.target_node
|
|
29
28
|
)
|
|
30
29
|
|
|
31
|
-
|
|
30
|
+
log.debug(f"Verifying envelope: {unsigned_envelope.model_dump_json(exclude_none=True)}")
|
|
32
31
|
|
|
33
32
|
pub_key.verify(
|
|
34
33
|
self.signature,
|
|
@@ -43,8 +42,8 @@ class UnsignedEnvelope(BaseModel, Generic[T]):
|
|
|
43
42
|
target_node: KoiNetNode
|
|
44
43
|
|
|
45
44
|
def sign_with(self, priv_key: PrivateKey) -> SignedEnvelope[T]:
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
log.debug(f"Signing envelope: {self.model_dump_json(exclude_none=True)}")
|
|
46
|
+
log.debug(f"Type: [{type(self.payload)}]")
|
|
48
47
|
|
|
49
48
|
signature = priv_key.sign(
|
|
50
49
|
self.model_dump_json(exclude_none=True).encode()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from typing import NamedTuple
|
|
2
|
+
from pydantic import BaseModel
|
|
3
|
+
from .envelope import SignedEnvelope
|
|
4
|
+
from .consts import (
|
|
5
|
+
BROADCAST_EVENTS_PATH,
|
|
6
|
+
POLL_EVENTS_PATH,
|
|
7
|
+
FETCH_BUNDLES_PATH,
|
|
8
|
+
FETCH_MANIFESTS_PATH,
|
|
9
|
+
FETCH_RIDS_PATH
|
|
10
|
+
)
|
|
11
|
+
from .api_models import (
|
|
12
|
+
EventsPayload,
|
|
13
|
+
PollEvents,
|
|
14
|
+
FetchBundles,
|
|
15
|
+
BundlesPayload,
|
|
16
|
+
FetchManifests,
|
|
17
|
+
ManifestsPayload,
|
|
18
|
+
FetchRids,
|
|
19
|
+
RidsPayload
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Models(NamedTuple):
|
|
24
|
+
request: type[BaseModel]
|
|
25
|
+
response: type[BaseModel] | None
|
|
26
|
+
request_envelope: type[SignedEnvelope]
|
|
27
|
+
response_envelope: type[SignedEnvelope] | None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
API_MODEL_MAP: dict[str, Models] = {
|
|
31
|
+
BROADCAST_EVENTS_PATH: Models(
|
|
32
|
+
request=EventsPayload,
|
|
33
|
+
response=None,
|
|
34
|
+
request_envelope=SignedEnvelope[EventsPayload],
|
|
35
|
+
response_envelope=None
|
|
36
|
+
),
|
|
37
|
+
POLL_EVENTS_PATH: Models(
|
|
38
|
+
request=PollEvents,
|
|
39
|
+
response=EventsPayload,
|
|
40
|
+
request_envelope=SignedEnvelope[PollEvents],
|
|
41
|
+
response_envelope=SignedEnvelope[EventsPayload]
|
|
42
|
+
),
|
|
43
|
+
FETCH_BUNDLES_PATH: Models(
|
|
44
|
+
request=FetchBundles,
|
|
45
|
+
response=BundlesPayload,
|
|
46
|
+
request_envelope=SignedEnvelope[FetchBundles],
|
|
47
|
+
response_envelope=SignedEnvelope[BundlesPayload]
|
|
48
|
+
),
|
|
49
|
+
FETCH_MANIFESTS_PATH: Models(
|
|
50
|
+
request=FetchManifests,
|
|
51
|
+
response=ManifestsPayload,
|
|
52
|
+
request_envelope=SignedEnvelope[FetchManifests],
|
|
53
|
+
response_envelope=SignedEnvelope[ManifestsPayload]
|
|
54
|
+
),
|
|
55
|
+
FETCH_RIDS_PATH: Models(
|
|
56
|
+
request=FetchRids,
|
|
57
|
+
response=RidsPayload,
|
|
58
|
+
request_envelope=SignedEnvelope[FetchRids],
|
|
59
|
+
response_envelope=SignedEnvelope[RidsPayload]
|
|
60
|
+
)
|
|
61
|
+
}
|
koi_net/protocol/node.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from enum import StrEnum
|
|
2
|
-
from pydantic import BaseModel
|
|
2
|
+
from pydantic import BaseModel
|
|
3
3
|
from rid_lib import RIDType
|
|
4
4
|
|
|
5
5
|
|
|
@@ -8,8 +8,8 @@ class NodeType(StrEnum):
|
|
|
8
8
|
PARTIAL = "PARTIAL"
|
|
9
9
|
|
|
10
10
|
class NodeProvides(BaseModel):
|
|
11
|
-
event: list[RIDType] =
|
|
12
|
-
state: list[RIDType] =
|
|
11
|
+
event: list[RIDType] = []
|
|
12
|
+
state: list[RIDType] = []
|
|
13
13
|
|
|
14
14
|
class NodeProfile(BaseModel):
|
|
15
15
|
base_url: str | None = None
|
koi_net/protocol/secure.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
from rid_lib.types import KoiNetNode
|
|
2
|
+
import structlog
|
|
2
3
|
from base64 import b64decode, b64encode
|
|
3
4
|
from cryptography.hazmat.primitives import hashes
|
|
4
5
|
from cryptography.hazmat.primitives.asymmetric import ec
|
|
@@ -9,7 +10,7 @@ from cryptography.hazmat.primitives.asymmetric.utils import (
|
|
|
9
10
|
encode_dss_signature
|
|
10
11
|
)
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
log = structlog.stdlib.get_logger()
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
def der_to_raw_signature(der_signature: bytes, curve=ec.SECP256R1()) -> bytes:
|
|
@@ -91,9 +92,9 @@ class PrivateKey:
|
|
|
91
92
|
|
|
92
93
|
signature = b64encode(raw_signature_bytes).decode()
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
log.debug(f"Signing message with [{self.public_key().to_der()}]")
|
|
96
|
+
log.debug(f"hash: {hashed_message}")
|
|
97
|
+
log.debug(f"signature: {signature}")
|
|
97
98
|
|
|
98
99
|
return signature
|
|
99
100
|
|
|
@@ -134,6 +135,11 @@ class PublicKey:
|
|
|
134
135
|
)
|
|
135
136
|
).decode()
|
|
136
137
|
|
|
138
|
+
def to_node_rid(self, name) -> KoiNetNode:
|
|
139
|
+
return KoiNetNode(
|
|
140
|
+
name=name,
|
|
141
|
+
hash=sha256_hash(self.to_der())
|
|
142
|
+
)
|
|
137
143
|
|
|
138
144
|
def verify(self, signature: str, message: bytes) -> bool:
|
|
139
145
|
# hashed_message = sha256_hash(message.decode())
|
|
@@ -144,9 +150,9 @@ class PublicKey:
|
|
|
144
150
|
# print()
|
|
145
151
|
# print(message.decode())
|
|
146
152
|
|
|
147
|
-
#
|
|
148
|
-
#
|
|
149
|
-
#
|
|
153
|
+
# log.debug(f"Verifying message with [{self.to_der()}]")
|
|
154
|
+
# log.debug(f"hash: {hashed_message}")
|
|
155
|
+
# log.debug(f"signature: {signature}")
|
|
150
156
|
|
|
151
157
|
raw_signature_bytes = b64decode(signature)
|
|
152
158
|
der_signature_bytes = raw_to_der_signature(raw_signature_bytes)
|
koi_net/secure.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import structlog
|
|
2
2
|
from functools import wraps
|
|
3
|
-
|
|
4
3
|
import cryptography.exceptions
|
|
5
4
|
from rid_lib.ext import Bundle, Cache
|
|
6
5
|
from rid_lib.ext.utils import sha256_hash
|
|
@@ -18,9 +17,9 @@ from .protocol.errors import (
|
|
|
18
17
|
InvalidSignatureError,
|
|
19
18
|
InvalidTargetError
|
|
20
19
|
)
|
|
21
|
-
from .config import NodeConfig
|
|
20
|
+
from .config.core import NodeConfig
|
|
22
21
|
|
|
23
|
-
|
|
22
|
+
log = structlog.stdlib.get_logger()
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
class Secure:
|
|
@@ -120,15 +119,15 @@ class Secure:
|
|
|
120
119
|
"""
|
|
121
120
|
@wraps(func)
|
|
122
121
|
async def wrapper(req: SignedEnvelope, *args, **kwargs) -> SignedEnvelope | None:
|
|
123
|
-
|
|
122
|
+
log.info("Validating envelope")
|
|
124
123
|
|
|
125
124
|
self.validate_envelope(req)
|
|
126
|
-
|
|
125
|
+
log.info("Calling endpoint handler")
|
|
127
126
|
|
|
128
127
|
result = await func(req, *args, **kwargs)
|
|
129
128
|
|
|
130
129
|
if result is not None:
|
|
131
|
-
|
|
130
|
+
log.info("Creating response envelope")
|
|
132
131
|
return self.create_envelope(
|
|
133
132
|
payload=result,
|
|
134
133
|
target=req.source_node
|
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import queue
|
|
2
2
|
import traceback
|
|
3
3
|
import time
|
|
4
|
-
import
|
|
4
|
+
import structlog
|
|
5
5
|
|
|
6
6
|
from rid_lib.ext import Cache
|
|
7
7
|
from rid_lib.types import KoiNetNode
|
|
8
8
|
|
|
9
|
-
from
|
|
10
|
-
from
|
|
11
|
-
from
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
from
|
|
16
|
-
from koi_net.worker import ThreadWorker
|
|
9
|
+
from ..config.core import NodeConfig
|
|
10
|
+
from ..network.event_queue import EventQueue, QueuedEvent
|
|
11
|
+
from ..network.request_handler import RequestHandler
|
|
12
|
+
from ..network.poll_event_buffer import PollEventBuffer
|
|
13
|
+
from ..protocol.event import Event
|
|
14
|
+
from ..protocol.node import NodeProfile, NodeType
|
|
15
|
+
from .base import ThreadWorker, STOP_WORKER
|
|
17
16
|
|
|
18
|
-
|
|
17
|
+
log = structlog.stdlib.get_logger()
|
|
19
18
|
|
|
20
19
|
|
|
21
20
|
class EventProcessingWorker(ThreadWorker):
|
|
@@ -28,10 +27,7 @@ class EventProcessingWorker(ThreadWorker):
|
|
|
28
27
|
request_handler: RequestHandler,
|
|
29
28
|
config: NodeConfig,
|
|
30
29
|
cache: Cache,
|
|
31
|
-
poll_event_buf: PollEventBuffer
|
|
32
|
-
queue_timeout: float = 0.1,
|
|
33
|
-
max_buf_len: int = 5,
|
|
34
|
-
max_wait_time: float = 1.0
|
|
30
|
+
poll_event_buf: PollEventBuffer
|
|
35
31
|
):
|
|
36
32
|
self.event_queue = event_queue
|
|
37
33
|
self.request_handler = request_handler
|
|
@@ -40,9 +36,9 @@ class EventProcessingWorker(ThreadWorker):
|
|
|
40
36
|
self.cache = cache
|
|
41
37
|
self.poll_event_buf = poll_event_buf
|
|
42
38
|
|
|
43
|
-
self.timeout =
|
|
44
|
-
self.max_buf_len =
|
|
45
|
-
self.max_wait_time =
|
|
39
|
+
self.timeout: float = 0.1
|
|
40
|
+
self.max_buf_len: int = 5
|
|
41
|
+
self.max_wait_time: float = 1.0
|
|
46
42
|
|
|
47
43
|
self.event_buffer = dict()
|
|
48
44
|
self.buffer_times = dict()
|
|
@@ -67,32 +63,32 @@ class EventProcessingWorker(ThreadWorker):
|
|
|
67
63
|
return True
|
|
68
64
|
|
|
69
65
|
elif node_profile.node_type == NodeType.PARTIAL:
|
|
70
|
-
self.poll_event_buf.
|
|
66
|
+
self.poll_event_buf.push(item.target, item.event)
|
|
71
67
|
return False
|
|
72
68
|
|
|
73
69
|
elif item.target == self.config.koi_net.first_contact.rid:
|
|
74
70
|
return True
|
|
75
71
|
|
|
76
72
|
else:
|
|
77
|
-
|
|
73
|
+
log.warning(f"Couldn't handle event {item.event!r} in queue, node {item.target!r} unknown to me")
|
|
78
74
|
return False
|
|
79
75
|
|
|
80
76
|
|
|
81
77
|
def run(self):
|
|
82
|
-
|
|
78
|
+
log.info("Started event worker")
|
|
83
79
|
while True:
|
|
84
80
|
now = time.time()
|
|
85
81
|
try:
|
|
86
82
|
item = self.event_queue.q.get(timeout=self.timeout)
|
|
87
83
|
|
|
88
84
|
try:
|
|
89
|
-
if item is
|
|
90
|
-
|
|
85
|
+
if item is STOP_WORKER:
|
|
86
|
+
log.info(f"Received 'STOP_WORKER' signal, flushing buffer...")
|
|
91
87
|
for target in self.event_buffer.keys():
|
|
92
88
|
self.flush_buffer(target, self.event_buffer[target])
|
|
93
89
|
return
|
|
94
90
|
|
|
95
|
-
|
|
91
|
+
log.info(f"Dequeued {item.event!r} -> {item.target!r}")
|
|
96
92
|
|
|
97
93
|
if not self.decide_event(item):
|
|
98
94
|
continue
|
|
@@ -1,38 +1,37 @@
|
|
|
1
1
|
import queue
|
|
2
2
|
import traceback
|
|
3
|
-
import
|
|
3
|
+
import structlog
|
|
4
4
|
|
|
5
|
-
from
|
|
6
|
-
from
|
|
7
|
-
from
|
|
8
|
-
from koi_net.worker import ThreadWorker
|
|
5
|
+
from ..processor.pipeline import KnowledgePipeline
|
|
6
|
+
from ..processor.kobj_queue import KobjQueue
|
|
7
|
+
from .base import ThreadWorker, STOP_WORKER
|
|
9
8
|
|
|
10
|
-
|
|
9
|
+
log = structlog.stdlib.get_logger()
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
class KnowledgeProcessingWorker(ThreadWorker):
|
|
14
13
|
def __init__(
|
|
15
14
|
self,
|
|
16
15
|
kobj_queue: KobjQueue,
|
|
17
|
-
pipeline: KnowledgePipeline
|
|
18
|
-
timeout: float = 0.1
|
|
16
|
+
pipeline: KnowledgePipeline
|
|
19
17
|
):
|
|
20
18
|
self.kobj_queue = kobj_queue
|
|
21
19
|
self.pipeline = pipeline
|
|
22
|
-
self.timeout =
|
|
20
|
+
self.timeout: float = 0.1
|
|
21
|
+
|
|
23
22
|
super().__init__()
|
|
24
23
|
|
|
25
24
|
def run(self):
|
|
26
|
-
|
|
25
|
+
log.info("Started kobj worker")
|
|
27
26
|
while True:
|
|
28
27
|
try:
|
|
29
28
|
item = self.kobj_queue.q.get(timeout=self.timeout)
|
|
30
29
|
try:
|
|
31
|
-
if item is
|
|
32
|
-
|
|
30
|
+
if item is STOP_WORKER:
|
|
31
|
+
log.info("Received 'STOP_WORKER' signal, shutting down...")
|
|
33
32
|
return
|
|
34
33
|
|
|
35
|
-
|
|
34
|
+
log.info(f"Dequeued {item!r}")
|
|
36
35
|
|
|
37
36
|
self.pipeline.process(item)
|
|
38
37
|
finally:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: koi-net
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.0b3
|
|
4
4
|
Summary: Implementation of KOI-net protocol in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/BlockScience/koi-net/
|
|
6
6
|
Author-email: Luke Miller <luke@block.science>
|
|
@@ -36,6 +36,7 @@ Requires-Dist: python-dotenv>=1.1.0
|
|
|
36
36
|
Requires-Dist: rich>=14.1.0
|
|
37
37
|
Requires-Dist: rid-lib>=3.2.7
|
|
38
38
|
Requires-Dist: ruamel-yaml>=0.18.10
|
|
39
|
+
Requires-Dist: structlog>=25.4.0
|
|
39
40
|
Requires-Dist: uvicorn>=0.34.2
|
|
40
41
|
Provides-Extra: dev
|
|
41
42
|
Requires-Dist: build; extra == 'dev'
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
koi_net/__init__.py,sha256=9z9lpKDuE7KF-evTBkAZechSvL9eQL4p0-ksmwZwMMU,21
|
|
2
|
+
koi_net/assembler.py,sha256=XR-e0-XHlrbXrZPJoO0tKaPH2xJ80KckfcjOeu9qr4o,3229
|
|
3
|
+
koi_net/core.py,sha256=gnR7moGHYkXHUxoV-mJRxi-tKZuPqvG9lg0jiVfAjXg,2257
|
|
4
|
+
koi_net/default_actions.py,sha256=2y7f9rUaENYRO8traTZbEQ7ySFZP_P8pUqf82bZj8Gk,602
|
|
5
|
+
koi_net/effector.py,sha256=fZEIBtvQQZmJx6HIcRB0jjP_EE00GREWuMQJ5m0uXOc,4937
|
|
6
|
+
koi_net/handshaker.py,sha256=d6s2R2eYxbOfpvRafKHNb_dD7LRdTkF4IsqFu7jQ-sQ,1211
|
|
7
|
+
koi_net/identity.py,sha256=-5YYda4xQ78EeGBspN9whWTuWttsUhZ8gxT2oLDEkpI,574
|
|
8
|
+
koi_net/lifecycle.py,sha256=axmr9KfEXzDT5HMVTVC6komMm9pdPQcODkudp7xVZ1k,4750
|
|
9
|
+
koi_net/logger.py,sha256=EzpQNvNrbi5LmMCobyuN1_wOz5VuMDeaPbLcyMrLKGI,6172
|
|
10
|
+
koi_net/secure.py,sha256=SUJqXScoSXgBs5PeeKtRKyDqGf35tGeNhXhwFa61IXE,4915
|
|
11
|
+
koi_net/utils.py,sha256=4K6BqtifExDUfPP8zJAICgPV1mcabz_MaFB0P0fSnic,411
|
|
12
|
+
koi_net/cli/__init__.py,sha256=Vpq0yLN0ROyF0qQYsyiuWFbjs-zsa90G1DpAv13qaiw,25
|
|
13
|
+
koi_net/cli/commands.py,sha256=5eVsvJAnHjwJdt8MNYrk5zSzWWm1ooiP35bJeQxYIq4,2592
|
|
14
|
+
koi_net/cli/models.py,sha256=9x66SSmIezw61KFbdk7VNoPQTi9JQGxtIlfWMNt5ZfU,1158
|
|
15
|
+
koi_net/config/core.py,sha256=RogkUeDHoQzpvZugqqk_8BkJiGVJX8ekLAZwYXtmsxs,2256
|
|
16
|
+
koi_net/config/full_node.py,sha256=sVxraN_fyM_jFHBhG7LEALHOlYbRvPIONuci8ws_bcA,945
|
|
17
|
+
koi_net/config/loader.py,sha256=3vFVz0V8pv6Ohq9pSMTpa40stYOlFB2do-aBKo7Wr8c,1311
|
|
18
|
+
koi_net/config/partial_node.py,sha256=ALfYXJ4lrMFhKt1nAr-Zn8HBVw9YLBSbN9ykEdwc91E,497
|
|
19
|
+
koi_net/entrypoints/__init__.py,sha256=DdKtM7uXYYZLDaHkzYVOVQh-g2S3bG0xdGAFXok6tj8,61
|
|
20
|
+
koi_net/entrypoints/base.py,sha256=nuaWRD4_0u5Cmieq4MiiJ2NrwOnoC4iaP4ulXXiInmI,85
|
|
21
|
+
koi_net/entrypoints/poller.py,sha256=5oo5Zj2fD43qkE7ai1M_PQi9vxDGbXPoZ4pAjL2H8X4,1472
|
|
22
|
+
koi_net/entrypoints/server.py,sha256=gbmRnSPo-OEaK2TyzTJ_1t-DlmtyARNSiqb07tq74bE,3090
|
|
23
|
+
koi_net/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
koi_net/network/error_handler.py,sha256=ZH0Py-XuSsnQiSB6rVZ2pywvMcKHr_VywnAiTI8RKEw,1780
|
|
25
|
+
koi_net/network/event_queue.py,sha256=vPrC5-QS0s57ALjdt2ddhNcSMwbvGakCvCKBMx4U6pw,734
|
|
26
|
+
koi_net/network/graph.py,sha256=ZJ1CY-SnYhnZAMm3e9YCbBflk80ZDETnoQKaBk5kLgo,4161
|
|
27
|
+
koi_net/network/poll_event_buffer.py,sha256=Wh4JcGnFaTHbveCzLWcqOzXAU_yk9EkUsukX5hI3pn8,735
|
|
28
|
+
koi_net/network/request_handler.py,sha256=XSX-JAqRua42YOakmia--9NnKcT32Fpoc-7aYfjDpwg,6752
|
|
29
|
+
koi_net/network/resolver.py,sha256=wg9yyJkZbS1G251u4tVav9SeIpH_Aq7FE0rLPvExn3g,5282
|
|
30
|
+
koi_net/network/response_handler.py,sha256=6GewpouDqMl1Ozgfkro_EcUqeJZEfY7msi1U7TQigCY,4212
|
|
31
|
+
koi_net/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
+
koi_net/processor/context.py,sha256=64rUQ6Oy3wbA5CtH40hqW7_ZPVDj4Wzw7dtVC0yIxkY,1323
|
|
33
|
+
koi_net/processor/handler.py,sha256=3z1PreLRH_48HU6o9lUD_5CAOgA8O9PuSB9iAusVa_c,2510
|
|
34
|
+
koi_net/processor/knowledge_handlers.py,sha256=H0DTHg2JI4_FvpafHjTMFC2mjjtgTCAew1s7V_fzajE,10908
|
|
35
|
+
koi_net/processor/knowledge_object.py,sha256=NJBNxzYdt_JNYpA71E1vaOuZNIKrkxRpK8l459mjbnc,4129
|
|
36
|
+
koi_net/processor/kobj_queue.py,sha256=tM_ssv2t_2t2A0w_X6rq-Gfxzl0NGCjt387dqTGxKxM,1833
|
|
37
|
+
koi_net/processor/pipeline.py,sha256=4CkOMOXyJ4J17Y5ulnlqJVnINjYqCGI1MBRz2oKylRM,8658
|
|
38
|
+
koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
|
+
koi_net/protocol/api_models.py,sha256=lPd_w-sncyrQlTQ__iae9Li1lFARu8nKiMMlXu7-Hls,1856
|
|
40
|
+
koi_net/protocol/consts.py,sha256=bisbVEojPIHlLhkLafBzfIhH25TjNfvTORF1g6YXzIM,243
|
|
41
|
+
koi_net/protocol/edge.py,sha256=PzdEhC43T1KO5iMSEu7I4tiz-7sZxtz41dJfWf-oHA0,1034
|
|
42
|
+
koi_net/protocol/envelope.py,sha256=4SKF4tP1P1ex7UEmYOJt7IP9_8fCWsa_o6ZBOu8SOKk,1877
|
|
43
|
+
koi_net/protocol/errors.py,sha256=uKPQ-TGLouZuK0xd2pXuCQoRTyu_JFsydSCLml13Cz8,595
|
|
44
|
+
koi_net/protocol/event.py,sha256=HxzLN-iCXPyr2YzrswMIkgZYeUdFbBpa5v98dAB06lQ,1328
|
|
45
|
+
koi_net/protocol/model_map.py,sha256=W0quo25HqVkzp8HQ7l_FNTtN4ZX0u7MIqp68Ir81fuk,1671
|
|
46
|
+
koi_net/protocol/node.py,sha256=7GQzHORFr9cP4BqJgir6EGSWCskL-yqmvJksIiLfcWU,409
|
|
47
|
+
koi_net/protocol/secure.py,sha256=_JRwPLLqifQQG_Z8QGeHfICX-pvOsppGvDy6_q8U8I8,5298
|
|
48
|
+
koi_net/workers/__init__.py,sha256=cTCXr4X_ONobHJxN0H8x3hvMgDpQnt7AeDbvcWzyz6Y,98
|
|
49
|
+
koi_net/workers/base.py,sha256=IvtXBTVmthUrURmB9Cn9wffeVv9_Cphsk97_GCLb5NQ,294
|
|
50
|
+
koi_net/workers/event_worker.py,sha256=Z-sShvc0jY3bNqN_P3INhAXFNKWP02YAyhzIFX984lk,4140
|
|
51
|
+
koi_net/workers/kobj_worker.py,sha256=VKGZZZMzSjp1NtuNGCUMmOxRFL0YwbLpTd6XlkQWfF0,1239
|
|
52
|
+
koi_net-1.2.0b3.dist-info/METADATA,sha256=PM-J7K9mH88xHgSeuDpcawOAQU8J8W3BqUJJ90cTUGY,37347
|
|
53
|
+
koi_net-1.2.0b3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
54
|
+
koi_net-1.2.0b3.dist-info/entry_points.txt,sha256=l7He_JTyXrfKIHkttnPWXHI717v8WpLLfCduL5QcEWA,40
|
|
55
|
+
koi_net-1.2.0b3.dist-info/licenses/LICENSE,sha256=03mgCL5qth2aD9C3F3qNVs4sFJSpK9kjtYCyOwdSp7s,1069
|
|
56
|
+
koi_net-1.2.0b3.dist-info/RECORD,,
|