koi-net 1.0.0b19__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of koi-net might be problematic. Click here for more details.
- koi_net/actor.py +60 -0
- koi_net/config.py +44 -18
- koi_net/context.py +63 -0
- koi_net/core.py +152 -84
- koi_net/default_actions.py +15 -0
- koi_net/effector.py +139 -0
- koi_net/identity.py +4 -22
- koi_net/lifecycle.py +104 -0
- koi_net/network/__init__.py +0 -1
- koi_net/network/error_handler.py +50 -0
- koi_net/network/event_queue.py +199 -0
- koi_net/network/graph.py +23 -38
- koi_net/network/request_handler.py +129 -66
- koi_net/network/resolver.py +150 -0
- koi_net/network/response_handler.py +15 -6
- koi_net/poller.py +40 -0
- koi_net/processor/__init__.py +0 -1
- koi_net/processor/default_handlers.py +71 -42
- koi_net/processor/handler.py +3 -7
- koi_net/processor/interface.py +15 -214
- koi_net/processor/knowledge_object.py +10 -17
- koi_net/processor/knowledge_pipeline.py +220 -0
- koi_net/protocol/api_models.py +18 -3
- koi_net/protocol/edge.py +26 -1
- koi_net/protocol/envelope.py +58 -0
- koi_net/protocol/errors.py +23 -0
- koi_net/protocol/event.py +0 -3
- koi_net/protocol/node.py +2 -1
- koi_net/protocol/secure.py +160 -0
- koi_net/secure.py +117 -0
- koi_net/server.py +129 -0
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/METADATA +5 -4
- koi_net-1.1.0.dist-info/RECORD +38 -0
- koi_net/network/interface.py +0 -276
- koi_net/protocol/helpers.py +0 -25
- koi_net-1.0.0b19.dist-info/RECORD +0 -25
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/WHEEL +0 -0
- {koi_net-1.0.0b19.dist-info → koi_net-1.1.0.dist-info}/licenses/LICENSE +0 -0
koi_net/secure.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from functools import wraps
|
|
3
|
+
|
|
4
|
+
import cryptography.exceptions
|
|
5
|
+
from rid_lib.ext import Bundle
|
|
6
|
+
from rid_lib.ext.utils import sha256_hash
|
|
7
|
+
from .identity import NodeIdentity
|
|
8
|
+
from .protocol.envelope import UnsignedEnvelope, SignedEnvelope
|
|
9
|
+
from .protocol.secure import PublicKey
|
|
10
|
+
from .protocol.api_models import EventsPayload
|
|
11
|
+
from .protocol.event import EventType
|
|
12
|
+
from .protocol.node import NodeProfile
|
|
13
|
+
from .protocol.secure import PrivateKey
|
|
14
|
+
from .protocol.errors import (
|
|
15
|
+
UnknownNodeError,
|
|
16
|
+
InvalidKeyError,
|
|
17
|
+
InvalidSignatureError,
|
|
18
|
+
InvalidTargetError
|
|
19
|
+
)
|
|
20
|
+
from .effector import Effector
|
|
21
|
+
from .config import NodeConfig
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Secure:
|
|
27
|
+
identity: NodeIdentity
|
|
28
|
+
effector: Effector
|
|
29
|
+
config: NodeConfig
|
|
30
|
+
priv_key: PrivateKey
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
identity: NodeIdentity,
|
|
35
|
+
effector: Effector,
|
|
36
|
+
config: NodeConfig
|
|
37
|
+
):
|
|
38
|
+
self.identity = identity
|
|
39
|
+
self.effector = effector
|
|
40
|
+
self.config = config
|
|
41
|
+
|
|
42
|
+
self.priv_key = self._load_priv_key()
|
|
43
|
+
|
|
44
|
+
def _load_priv_key(self) -> PrivateKey:
|
|
45
|
+
with open(self.config.koi_net.private_key_pem_path, "r") as f:
|
|
46
|
+
priv_key_pem = f.read()
|
|
47
|
+
|
|
48
|
+
return PrivateKey.from_pem(
|
|
49
|
+
priv_key_pem=priv_key_pem,
|
|
50
|
+
password=self.config.env.priv_key_password
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def _handle_unknown_node(self, envelope: SignedEnvelope) -> Bundle | None:
|
|
54
|
+
if type(envelope.payload) != EventsPayload:
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
for event in envelope.payload.events:
|
|
58
|
+
# must be NEW event for bundle of source node's profile
|
|
59
|
+
if event.rid != envelope.source_node:
|
|
60
|
+
continue
|
|
61
|
+
if event.event_type != EventType.NEW:
|
|
62
|
+
continue
|
|
63
|
+
|
|
64
|
+
return event.bundle
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
def create_envelope(self, payload, target) -> SignedEnvelope:
|
|
68
|
+
return UnsignedEnvelope(
|
|
69
|
+
payload=payload,
|
|
70
|
+
source_node=self.identity.rid,
|
|
71
|
+
target_node=target
|
|
72
|
+
).sign_with(self.priv_key)
|
|
73
|
+
|
|
74
|
+
def validate_envelope(self, envelope: SignedEnvelope):
|
|
75
|
+
node_bundle = (
|
|
76
|
+
self.effector.deref(envelope.source_node) or
|
|
77
|
+
self._handle_unknown_node(envelope)
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if not node_bundle:
|
|
81
|
+
raise UnknownNodeError(f"Couldn't resolve {envelope.source_node}")
|
|
82
|
+
|
|
83
|
+
node_profile = node_bundle.validate_contents(NodeProfile)
|
|
84
|
+
|
|
85
|
+
# check that public key matches source node RID
|
|
86
|
+
if envelope.source_node.hash != sha256_hash(node_profile.public_key):
|
|
87
|
+
raise InvalidKeyError("Invalid public key on new node!")
|
|
88
|
+
|
|
89
|
+
# check envelope signed by validated public key
|
|
90
|
+
pub_key = PublicKey.from_der(node_profile.public_key)
|
|
91
|
+
try:
|
|
92
|
+
envelope.verify_with(pub_key)
|
|
93
|
+
except cryptography.exceptions.InvalidSignature as err:
|
|
94
|
+
raise InvalidSignatureError(f"Signature {envelope.signature} is invalid.")
|
|
95
|
+
|
|
96
|
+
# check that this node is the target of the envelope
|
|
97
|
+
if envelope.target_node != self.identity.rid:
|
|
98
|
+
raise InvalidTargetError(f"Envelope target {envelope.target_node!r} is not me")
|
|
99
|
+
|
|
100
|
+
def envelope_handler(self, func):
|
|
101
|
+
@wraps(func)
|
|
102
|
+
async def wrapper(req: SignedEnvelope, *args, **kwargs) -> SignedEnvelope | None:
|
|
103
|
+
logger.info("Validating envelope")
|
|
104
|
+
|
|
105
|
+
self.validate_envelope(req)
|
|
106
|
+
logger.info("Calling endpoint handler")
|
|
107
|
+
|
|
108
|
+
result = await func(req, *args, **kwargs)
|
|
109
|
+
|
|
110
|
+
if result is not None:
|
|
111
|
+
logger.info("Creating response envelope")
|
|
112
|
+
return self.create_envelope(
|
|
113
|
+
payload=result,
|
|
114
|
+
target=req.source_node
|
|
115
|
+
)
|
|
116
|
+
return wrapper
|
|
117
|
+
|
koi_net/server.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import uvicorn
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from fastapi import FastAPI, APIRouter
|
|
5
|
+
from fastapi.responses import JSONResponse
|
|
6
|
+
from .network.event_queue import NetworkEventQueue
|
|
7
|
+
from .network.response_handler import ResponseHandler
|
|
8
|
+
from .processor.interface import ProcessorInterface
|
|
9
|
+
from .protocol.api_models import (
|
|
10
|
+
PollEvents,
|
|
11
|
+
FetchRids,
|
|
12
|
+
FetchManifests,
|
|
13
|
+
FetchBundles,
|
|
14
|
+
EventsPayload,
|
|
15
|
+
RidsPayload,
|
|
16
|
+
ManifestsPayload,
|
|
17
|
+
BundlesPayload,
|
|
18
|
+
ErrorResponse
|
|
19
|
+
)
|
|
20
|
+
from .protocol.errors import ProtocolError
|
|
21
|
+
from .protocol.envelope import SignedEnvelope
|
|
22
|
+
from .protocol.consts import (
|
|
23
|
+
BROADCAST_EVENTS_PATH,
|
|
24
|
+
POLL_EVENTS_PATH,
|
|
25
|
+
FETCH_RIDS_PATH,
|
|
26
|
+
FETCH_MANIFESTS_PATH,
|
|
27
|
+
FETCH_BUNDLES_PATH
|
|
28
|
+
)
|
|
29
|
+
from .secure import Secure
|
|
30
|
+
from .lifecycle import NodeLifecycle
|
|
31
|
+
from .config import NodeConfig
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class NodeServer:
|
|
37
|
+
lifecycle: NodeLifecycle
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
config: NodeConfig,
|
|
42
|
+
lifecycle: NodeLifecycle,
|
|
43
|
+
secure: Secure,
|
|
44
|
+
processor: ProcessorInterface,
|
|
45
|
+
event_queue: NetworkEventQueue,
|
|
46
|
+
response_handler: ResponseHandler
|
|
47
|
+
):
|
|
48
|
+
self.config = config
|
|
49
|
+
self.lifecycle = lifecycle
|
|
50
|
+
self.secure = secure
|
|
51
|
+
self.processor = processor
|
|
52
|
+
self.event_queue = event_queue
|
|
53
|
+
self.response_handler = response_handler
|
|
54
|
+
self._build_app()
|
|
55
|
+
|
|
56
|
+
def _build_app(self):
|
|
57
|
+
|
|
58
|
+
@asynccontextmanager
|
|
59
|
+
async def lifespan(*args, **kwargs):
|
|
60
|
+
async with self.lifecycle.async_run():
|
|
61
|
+
yield
|
|
62
|
+
|
|
63
|
+
self.app = FastAPI(
|
|
64
|
+
lifespan=lifespan,
|
|
65
|
+
title="KOI-net Protocol API",
|
|
66
|
+
version="1.0.0"
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
self.router = APIRouter(prefix="/koi-net")
|
|
70
|
+
self.app.add_exception_handler(ProtocolError, self.protocol_error_handler)
|
|
71
|
+
|
|
72
|
+
def _add_endpoint(path, func):
|
|
73
|
+
self.router.add_api_route(
|
|
74
|
+
path=path,
|
|
75
|
+
endpoint=self.secure.envelope_handler(func),
|
|
76
|
+
methods=["POST"],
|
|
77
|
+
response_model_exclude_none=True
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
_add_endpoint(BROADCAST_EVENTS_PATH, self.broadcast_events)
|
|
81
|
+
_add_endpoint(POLL_EVENTS_PATH, self.poll_events)
|
|
82
|
+
_add_endpoint(FETCH_RIDS_PATH, self.fetch_rids)
|
|
83
|
+
_add_endpoint(FETCH_MANIFESTS_PATH, self.fetch_manifests)
|
|
84
|
+
_add_endpoint(FETCH_BUNDLES_PATH, self.fetch_bundles)
|
|
85
|
+
|
|
86
|
+
self.app.include_router(self.router)
|
|
87
|
+
|
|
88
|
+
def run(self):
|
|
89
|
+
uvicorn.run(
|
|
90
|
+
app=self.app,
|
|
91
|
+
host=self.config.server.host,
|
|
92
|
+
port=self.config.server.port
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def protocol_error_handler(self, request, exc: ProtocolError):
|
|
96
|
+
logger.info(f"caught protocol error: {exc}")
|
|
97
|
+
resp = ErrorResponse(error=exc.error_type)
|
|
98
|
+
logger.info(f"returning error response: {resp}")
|
|
99
|
+
return JSONResponse(
|
|
100
|
+
status_code=400,
|
|
101
|
+
content=resp.model_dump(mode="json")
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
async def broadcast_events(self, req: SignedEnvelope[EventsPayload]):
|
|
105
|
+
logger.info(f"Request to {BROADCAST_EVENTS_PATH}, received {len(req.payload.events)} event(s)")
|
|
106
|
+
for event in req.payload.events:
|
|
107
|
+
self.processor.handle(event=event, source=req.source_node)
|
|
108
|
+
|
|
109
|
+
async def poll_events(
|
|
110
|
+
self, req: SignedEnvelope[PollEvents]
|
|
111
|
+
) -> SignedEnvelope[EventsPayload] | ErrorResponse:
|
|
112
|
+
logger.info(f"Request to {POLL_EVENTS_PATH}")
|
|
113
|
+
events = self.event_queue.flush_poll_queue(req.source_node)
|
|
114
|
+
return EventsPayload(events=events)
|
|
115
|
+
|
|
116
|
+
async def fetch_rids(
|
|
117
|
+
self, req: SignedEnvelope[FetchRids]
|
|
118
|
+
) -> SignedEnvelope[RidsPayload] | ErrorResponse:
|
|
119
|
+
return self.response_handler.fetch_rids(req.payload, req.source_node)
|
|
120
|
+
|
|
121
|
+
async def fetch_manifests(
|
|
122
|
+
self, req: SignedEnvelope[FetchManifests]
|
|
123
|
+
) -> SignedEnvelope[ManifestsPayload] | ErrorResponse:
|
|
124
|
+
return self.response_handler.fetch_manifests(req.payload, req.source_node)
|
|
125
|
+
|
|
126
|
+
async def fetch_bundles(
|
|
127
|
+
self, req: SignedEnvelope[FetchBundles]
|
|
128
|
+
) -> SignedEnvelope[BundlesPayload] | ErrorResponse:
|
|
129
|
+
return self.response_handler.fetch_bundles(req.payload, req.source_node)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: koi-net
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Implementation of KOI-net protocol in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/BlockScience/koi-net/
|
|
6
6
|
Author-email: Luke Miller <luke@block.science>
|
|
@@ -27,19 +27,20 @@ License: MIT License
|
|
|
27
27
|
SOFTWARE.
|
|
28
28
|
License-File: LICENSE
|
|
29
29
|
Requires-Python: >=3.10
|
|
30
|
+
Requires-Dist: cryptography>=45.0.3
|
|
31
|
+
Requires-Dist: fastapi>=0.115.12
|
|
30
32
|
Requires-Dist: httpx>=0.28.1
|
|
31
33
|
Requires-Dist: networkx>=3.4.2
|
|
32
34
|
Requires-Dist: pydantic>=2.10.6
|
|
33
35
|
Requires-Dist: python-dotenv>=1.1.0
|
|
34
|
-
Requires-Dist: rid-lib>=3.2.
|
|
36
|
+
Requires-Dist: rid-lib>=3.2.7
|
|
35
37
|
Requires-Dist: ruamel-yaml>=0.18.10
|
|
38
|
+
Requires-Dist: uvicorn>=0.34.2
|
|
36
39
|
Provides-Extra: dev
|
|
37
40
|
Requires-Dist: build; extra == 'dev'
|
|
38
41
|
Requires-Dist: twine>=6.0; extra == 'dev'
|
|
39
42
|
Provides-Extra: examples
|
|
40
|
-
Requires-Dist: fastapi; extra == 'examples'
|
|
41
43
|
Requires-Dist: rich; extra == 'examples'
|
|
42
|
-
Requires-Dist: uvicorn; extra == 'examples'
|
|
43
44
|
Description-Content-Type: text/markdown
|
|
44
45
|
|
|
45
46
|
# KOI-net
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
|
|
2
|
+
koi_net/actor.py,sha256=Gad1Xg8n_-zm6sj0nDkm_B1MwErQJKeUsbGmIDTRqJk,1820
|
|
3
|
+
koi_net/config.py,sha256=47XbQ59GRYFi4rlsoWKlnzMQATcnK70i3qmKTZAGOQk,4087
|
|
4
|
+
koi_net/context.py,sha256=G067ecIJJ5k8aesdyxjZC_vh3zVg9PR_H2U-09YIxXA,1683
|
|
5
|
+
koi_net/core.py,sha256=6nEDAOkMTv-pynU_hLOG4tKVWhopMPIHyZI6oJTdLj8,7061
|
|
6
|
+
koi_net/default_actions.py,sha256=TkQR9oj9CpO37Gb5bZLmFNl-Q8n3OxGiX4dvxQR7SaA,421
|
|
7
|
+
koi_net/effector.py,sha256=gSyZgRxQ91X04UL261e2pXWUfBHnQTGtjSHpc2JufxA,4097
|
|
8
|
+
koi_net/identity.py,sha256=FvIWksGTqwM7HCevIwmo_6l-t-2tnYkaaR4CanZatL4,569
|
|
9
|
+
koi_net/lifecycle.py,sha256=1WCo-VOox0rK3v_lfzhnFMGiyacZJpsaENlyGq7jHJQ,3563
|
|
10
|
+
koi_net/poller.py,sha256=bIrlqdac5vLQYAid35xiQJLDMR85GnOSPCXSTQ07-Mc,1173
|
|
11
|
+
koi_net/secure.py,sha256=cGNF2assqCaYq0i0fhQBm7aREoAdpY-XVypDsE1ALaU,3970
|
|
12
|
+
koi_net/server.py,sha256=PrR_cXQV5YMKa6FXwiJXwMZJ52VQVzLPYYPVl-Miuw8,4315
|
|
13
|
+
koi_net/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
koi_net/network/error_handler.py,sha256=09ulFPpSzoI49RFYPxc5TLXslqJQ_78q7TzLJIABeMw,1606
|
|
15
|
+
koi_net/network/event_queue.py,sha256=DWs26C235iYkP4koKcdbhmIOHGZRJ48d072BoNWyiHo,7325
|
|
16
|
+
koi_net/network/graph.py,sha256=neVVVHSyqsQR8hW2-lLC8IkDC2xciBXh3aWYXQpVqZs,4134
|
|
17
|
+
koi_net/network/request_handler.py,sha256=_SM5MuYkS636wGJeFkesapQsW5x_kt_1o9KTXB0wksU,6869
|
|
18
|
+
koi_net/network/resolver.py,sha256=YpQq6HKyfcoqr9TnHRnlQI33IM3tE0ljU02pZ3wWLh8,5410
|
|
19
|
+
koi_net/network/response_handler.py,sha256=__R_EvEpjaMz3PCDvkNgWF_EAHe2nePGk-zK_cT4C4g,2077
|
|
20
|
+
koi_net/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
koi_net/processor/default_handlers.py,sha256=1OTC4p0luTadNm90q6Fr_dbvysFzgRCbltp-YP6cRXo,9562
|
|
22
|
+
koi_net/processor/handler.py,sha256=_loaHjgVGVUxtCQdvAY9dQ0iqiq5co7wB2tK-usuv3Y,2355
|
|
23
|
+
koi_net/processor/interface.py,sha256=ebDwqggznFRfp2PT8-UJPUAvCwX8nZaaQ68FUeWQvmw,3682
|
|
24
|
+
koi_net/processor/knowledge_object.py,sha256=avQnsaeqqiJxy40P1VGljuQMtAGmJB-TBa4pmBXTaIs,3863
|
|
25
|
+
koi_net/processor/knowledge_pipeline.py,sha256=i7FpCFl0UIOwCI5zhP1i8M4PX4A48VN28iV9jruvN5k,9486
|
|
26
|
+
koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
|
+
koi_net/protocol/api_models.py,sha256=jzRZWW_ZB5YsBAiwCom882-WIbr0rPyelJxExRgHZGc,1755
|
|
28
|
+
koi_net/protocol/consts.py,sha256=bisbVEojPIHlLhkLafBzfIhH25TjNfvTORF1g6YXzIM,243
|
|
29
|
+
koi_net/protocol/edge.py,sha256=PzdEhC43T1KO5iMSEu7I4tiz-7sZxtz41dJfWf-oHA0,1034
|
|
30
|
+
koi_net/protocol/envelope.py,sha256=UVHlO2BDyDiP5eixqx9xD6xUsCfFRi0kZyzC4BC-DOw,1886
|
|
31
|
+
koi_net/protocol/errors.py,sha256=uKPQ-TGLouZuK0xd2pXuCQoRTyu_JFsydSCLml13Cz8,595
|
|
32
|
+
koi_net/protocol/event.py,sha256=HxzLN-iCXPyr2YzrswMIkgZYeUdFbBpa5v98dAB06lQ,1328
|
|
33
|
+
koi_net/protocol/node.py,sha256=7GQzHORFr9cP4BqJgir6EGSWCskL-yqmvJksIiLfcWU,409
|
|
34
|
+
koi_net/protocol/secure.py,sha256=6sRLWxG5EDF0QLBj29gk3hPmZnPXATrTTFdwx39wQfY,5127
|
|
35
|
+
koi_net-1.1.0.dist-info/METADATA,sha256=upjYpZiqyRhDFxl2xN_tNvDZ5Cx6G_Bs8dgnp5f6L0k,37116
|
|
36
|
+
koi_net-1.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
37
|
+
koi_net-1.1.0.dist-info/licenses/LICENSE,sha256=03mgCL5qth2aD9C3F3qNVs4sFJSpK9kjtYCyOwdSp7s,1069
|
|
38
|
+
koi_net-1.1.0.dist-info/RECORD,,
|
koi_net/network/interface.py
DELETED
|
@@ -1,276 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from queue import Queue
|
|
3
|
-
from typing import Generic
|
|
4
|
-
import httpx
|
|
5
|
-
from pydantic import BaseModel
|
|
6
|
-
from rid_lib import RID
|
|
7
|
-
from rid_lib.core import RIDType
|
|
8
|
-
from rid_lib.ext import Cache
|
|
9
|
-
from rid_lib.types import KoiNetNode
|
|
10
|
-
|
|
11
|
-
from .graph import NetworkGraph
|
|
12
|
-
from .request_handler import RequestHandler
|
|
13
|
-
from .response_handler import ResponseHandler
|
|
14
|
-
from ..protocol.node import NodeType
|
|
15
|
-
from ..protocol.edge import EdgeType
|
|
16
|
-
from ..protocol.event import Event
|
|
17
|
-
from ..identity import NodeIdentity
|
|
18
|
-
from ..config import ConfigType
|
|
19
|
-
|
|
20
|
-
logger = logging.getLogger(__name__)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class EventQueueModel(BaseModel):
|
|
24
|
-
webhook: dict[KoiNetNode, list[Event]]
|
|
25
|
-
poll: dict[KoiNetNode, list[Event]]
|
|
26
|
-
|
|
27
|
-
type EventQueue = dict[RID, Queue[Event]]
|
|
28
|
-
|
|
29
|
-
class NetworkInterface(Generic[ConfigType]):
|
|
30
|
-
"""A collection of functions and classes to interact with the KOI network."""
|
|
31
|
-
|
|
32
|
-
config: ConfigType
|
|
33
|
-
identity: NodeIdentity
|
|
34
|
-
cache: Cache
|
|
35
|
-
graph: NetworkGraph
|
|
36
|
-
request_handler: RequestHandler
|
|
37
|
-
response_handler: ResponseHandler
|
|
38
|
-
poll_event_queue: EventQueue
|
|
39
|
-
webhook_event_queue: EventQueue
|
|
40
|
-
|
|
41
|
-
def __init__(
|
|
42
|
-
self,
|
|
43
|
-
config: ConfigType,
|
|
44
|
-
cache: Cache,
|
|
45
|
-
identity: NodeIdentity
|
|
46
|
-
):
|
|
47
|
-
self.config = config
|
|
48
|
-
self.identity = identity
|
|
49
|
-
self.cache = cache
|
|
50
|
-
self.graph = NetworkGraph(cache, identity)
|
|
51
|
-
self.request_handler = RequestHandler(cache, self.graph)
|
|
52
|
-
self.response_handler = ResponseHandler(cache)
|
|
53
|
-
|
|
54
|
-
self.poll_event_queue = dict()
|
|
55
|
-
self.webhook_event_queue = dict()
|
|
56
|
-
self._load_event_queues()
|
|
57
|
-
|
|
58
|
-
def _load_event_queues(self):
|
|
59
|
-
"""Loads event queues from storage."""
|
|
60
|
-
try:
|
|
61
|
-
with open(self.config.koi_net.event_queues_path, "r") as f:
|
|
62
|
-
queues = EventQueueModel.model_validate_json(f.read())
|
|
63
|
-
|
|
64
|
-
for node in queues.poll.keys():
|
|
65
|
-
for event in queues.poll[node]:
|
|
66
|
-
queue = self.poll_event_queue.setdefault(node, Queue())
|
|
67
|
-
queue.put(event)
|
|
68
|
-
|
|
69
|
-
for node in queues.webhook.keys():
|
|
70
|
-
for event in queues.webhook[node]:
|
|
71
|
-
queue = self.webhook_event_queue.setdefault(node, Queue())
|
|
72
|
-
queue.put(event)
|
|
73
|
-
|
|
74
|
-
except FileNotFoundError:
|
|
75
|
-
return
|
|
76
|
-
|
|
77
|
-
def _save_event_queues(self):
|
|
78
|
-
"""Writes event queues to storage."""
|
|
79
|
-
events_model = EventQueueModel(
|
|
80
|
-
poll={
|
|
81
|
-
node: list(queue.queue)
|
|
82
|
-
for node, queue in self.poll_event_queue.items()
|
|
83
|
-
if not queue.empty()
|
|
84
|
-
},
|
|
85
|
-
webhook={
|
|
86
|
-
node: list(queue.queue)
|
|
87
|
-
for node, queue in self.webhook_event_queue.items()
|
|
88
|
-
if not queue.empty()
|
|
89
|
-
}
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
|
|
93
|
-
return
|
|
94
|
-
|
|
95
|
-
with open(self.config.koi_net.event_queues_path, "w") as f:
|
|
96
|
-
f.write(events_model.model_dump_json(indent=2))
|
|
97
|
-
|
|
98
|
-
def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
|
|
99
|
-
"""Pushes event to queue of specified node.
|
|
100
|
-
|
|
101
|
-
Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
|
|
102
|
-
"""
|
|
103
|
-
logger.debug(f"Pushing event {event.event_type} {event.rid} to {node}")
|
|
104
|
-
|
|
105
|
-
node_profile = self.graph.get_node_profile(node)
|
|
106
|
-
if not node_profile:
|
|
107
|
-
logger.warning(f"Node {node!r} unknown to me")
|
|
108
|
-
|
|
109
|
-
# if there's an edge from me to the target node, override broadcast type
|
|
110
|
-
edge_profile = self.graph.get_edge_profile(
|
|
111
|
-
source=self.identity.rid,
|
|
112
|
-
target=node
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
if edge_profile:
|
|
116
|
-
if edge_profile.edge_type == EdgeType.WEBHOOK:
|
|
117
|
-
event_queue = self.webhook_event_queue
|
|
118
|
-
elif edge_profile.edge_type == EdgeType.POLL:
|
|
119
|
-
event_queue = self.poll_event_queue
|
|
120
|
-
else:
|
|
121
|
-
if node_profile.node_type == NodeType.FULL:
|
|
122
|
-
event_queue = self.webhook_event_queue
|
|
123
|
-
elif node_profile.node_type == NodeType.PARTIAL:
|
|
124
|
-
event_queue = self.poll_event_queue
|
|
125
|
-
|
|
126
|
-
queue = event_queue.setdefault(node, Queue())
|
|
127
|
-
queue.put(event)
|
|
128
|
-
|
|
129
|
-
if flush and event_queue is self.webhook_event_queue:
|
|
130
|
-
self.flush_webhook_queue(node)
|
|
131
|
-
|
|
132
|
-
def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
|
|
133
|
-
"""Flushes a node's queue, returning list of events."""
|
|
134
|
-
queue = event_queue.get(node)
|
|
135
|
-
events = list()
|
|
136
|
-
if queue:
|
|
137
|
-
while not queue.empty():
|
|
138
|
-
event = queue.get()
|
|
139
|
-
logger.debug(f"Dequeued {event.event_type} '{event.rid}'")
|
|
140
|
-
events.append(event)
|
|
141
|
-
|
|
142
|
-
return events
|
|
143
|
-
|
|
144
|
-
def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
|
|
145
|
-
"""Flushes a node's poll queue, returning list of events."""
|
|
146
|
-
logger.debug(f"Flushing poll queue for {node}")
|
|
147
|
-
return self._flush_queue(self.poll_event_queue, node)
|
|
148
|
-
|
|
149
|
-
def flush_webhook_queue(self, node: KoiNetNode):
|
|
150
|
-
"""Flushes a node's webhook queue, and broadcasts events.
|
|
151
|
-
|
|
152
|
-
If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
|
|
153
|
-
"""
|
|
154
|
-
|
|
155
|
-
logger.debug(f"Flushing webhook queue for {node}")
|
|
156
|
-
|
|
157
|
-
node_profile = self.graph.get_node_profile(node)
|
|
158
|
-
|
|
159
|
-
if not node_profile:
|
|
160
|
-
logger.warning(f"{node!r} not found")
|
|
161
|
-
return
|
|
162
|
-
|
|
163
|
-
if node_profile.node_type != NodeType.FULL:
|
|
164
|
-
logger.warning(f"{node!r} is a partial node!")
|
|
165
|
-
return
|
|
166
|
-
|
|
167
|
-
events = self._flush_queue(self.webhook_event_queue, node)
|
|
168
|
-
if not events: return
|
|
169
|
-
|
|
170
|
-
logger.debug(f"Broadcasting {len(events)} events")
|
|
171
|
-
|
|
172
|
-
try:
|
|
173
|
-
self.request_handler.broadcast_events(node, events=events)
|
|
174
|
-
return True
|
|
175
|
-
except httpx.ConnectError:
|
|
176
|
-
logger.warning("Broadcast failed")
|
|
177
|
-
for event in events:
|
|
178
|
-
self.push_event_to(event, node)
|
|
179
|
-
return False
|
|
180
|
-
|
|
181
|
-
def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
|
|
182
|
-
"""Returns list of node RIDs which provide state for the specified RID type."""
|
|
183
|
-
|
|
184
|
-
logger.debug(f"Looking for state providers of '{rid_type}'")
|
|
185
|
-
provider_nodes = []
|
|
186
|
-
for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
|
|
187
|
-
node = self.graph.get_node_profile(node_rid)
|
|
188
|
-
|
|
189
|
-
if node.node_type == NodeType.FULL and rid_type in node.provides.state:
|
|
190
|
-
logger.debug(f"Found provider '{node_rid}'")
|
|
191
|
-
provider_nodes.append(node_rid)
|
|
192
|
-
|
|
193
|
-
if not provider_nodes:
|
|
194
|
-
logger.debug("Failed to find providers")
|
|
195
|
-
return provider_nodes
|
|
196
|
-
|
|
197
|
-
def fetch_remote_bundle(self, rid: RID):
|
|
198
|
-
"""Attempts to fetch a bundle by RID from known peer nodes."""
|
|
199
|
-
|
|
200
|
-
logger.debug(f"Fetching remote bundle '{rid}'")
|
|
201
|
-
remote_bundle = None
|
|
202
|
-
for node_rid in self.get_state_providers(type(rid)):
|
|
203
|
-
payload = self.request_handler.fetch_bundles(
|
|
204
|
-
node=node_rid, rids=[rid])
|
|
205
|
-
|
|
206
|
-
if payload.bundles:
|
|
207
|
-
remote_bundle = payload.bundles[0]
|
|
208
|
-
logger.debug(f"Got bundle from '{node_rid}'")
|
|
209
|
-
break
|
|
210
|
-
|
|
211
|
-
if not remote_bundle:
|
|
212
|
-
logger.warning("Failed to fetch remote bundle")
|
|
213
|
-
|
|
214
|
-
return remote_bundle
|
|
215
|
-
|
|
216
|
-
def fetch_remote_manifest(self, rid: RID):
|
|
217
|
-
"""Attempts to fetch a manifest by RID from known peer nodes."""
|
|
218
|
-
|
|
219
|
-
logger.debug(f"Fetching remote manifest '{rid}'")
|
|
220
|
-
remote_manifest = None
|
|
221
|
-
for node_rid in self.get_state_providers(type(rid)):
|
|
222
|
-
payload = self.request_handler.fetch_manifests(
|
|
223
|
-
node=node_rid, rids=[rid])
|
|
224
|
-
|
|
225
|
-
if payload.manifests:
|
|
226
|
-
remote_manifest = payload.manifests[0]
|
|
227
|
-
logger.debug(f"Got bundle from '{node_rid}'")
|
|
228
|
-
break
|
|
229
|
-
|
|
230
|
-
if not remote_manifest:
|
|
231
|
-
logger.warning("Failed to fetch remote bundle")
|
|
232
|
-
|
|
233
|
-
return remote_manifest
|
|
234
|
-
|
|
235
|
-
def poll_neighbors(self) -> list[Event]:
|
|
236
|
-
"""Polls all neighboring nodes and returns compiled list of events.
|
|
237
|
-
|
|
238
|
-
If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
|
|
239
|
-
"""
|
|
240
|
-
|
|
241
|
-
neighbors = self.graph.get_neighbors()
|
|
242
|
-
|
|
243
|
-
if not neighbors and self.config.koi_net.first_contact:
|
|
244
|
-
logger.debug("No neighbors found, polling first contact")
|
|
245
|
-
try:
|
|
246
|
-
payload = self.request_handler.poll_events(
|
|
247
|
-
url=self.config.koi_net.first_contact,
|
|
248
|
-
rid=self.identity.rid
|
|
249
|
-
)
|
|
250
|
-
if payload.events:
|
|
251
|
-
logger.debug(f"Received {len(payload.events)} events from '{self.config.koi_net.first_contact}'")
|
|
252
|
-
return payload.events
|
|
253
|
-
except httpx.ConnectError:
|
|
254
|
-
logger.debug(f"Failed to reach first contact '{self.config.koi_net.first_contact}'")
|
|
255
|
-
|
|
256
|
-
events = []
|
|
257
|
-
for node_rid in neighbors:
|
|
258
|
-
node = self.graph.get_node_profile(node_rid)
|
|
259
|
-
if not node: continue
|
|
260
|
-
if node.node_type != NodeType.FULL: continue
|
|
261
|
-
|
|
262
|
-
try:
|
|
263
|
-
payload = self.request_handler.poll_events(
|
|
264
|
-
node=node_rid,
|
|
265
|
-
rid=self.identity.rid
|
|
266
|
-
)
|
|
267
|
-
if payload.events:
|
|
268
|
-
logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
|
|
269
|
-
events.extend(payload.events)
|
|
270
|
-
except httpx.ConnectError:
|
|
271
|
-
logger.debug(f"Failed to reach node '{node_rid}'")
|
|
272
|
-
continue
|
|
273
|
-
|
|
274
|
-
return events
|
|
275
|
-
|
|
276
|
-
|
koi_net/protocol/helpers.py
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
from rid_lib.core import RIDType
|
|
2
|
-
from rid_lib.ext.bundle import Bundle
|
|
3
|
-
from rid_lib.types import KoiNetEdge
|
|
4
|
-
from rid_lib.types.koi_net_node import KoiNetNode
|
|
5
|
-
from .edge import EdgeProfile, EdgeStatus, EdgeType
|
|
6
|
-
|
|
7
|
-
def generate_edge_bundle(
|
|
8
|
-
source: KoiNetNode,
|
|
9
|
-
target: KoiNetNode,
|
|
10
|
-
rid_types: list[RIDType],
|
|
11
|
-
edge_type: EdgeType
|
|
12
|
-
) -> Bundle:
|
|
13
|
-
edge_rid = KoiNetEdge.generate(source, target)
|
|
14
|
-
edge_profile = EdgeProfile(
|
|
15
|
-
source=source,
|
|
16
|
-
target=target,
|
|
17
|
-
rid_types=rid_types,
|
|
18
|
-
edge_type=edge_type,
|
|
19
|
-
status=EdgeStatus.PROPOSED
|
|
20
|
-
)
|
|
21
|
-
edge_bundle = Bundle.generate(
|
|
22
|
-
edge_rid,
|
|
23
|
-
edge_profile.model_dump()
|
|
24
|
-
)
|
|
25
|
-
return edge_bundle
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
|
|
2
|
-
koi_net/config.py,sha256=TIKb1kFTcEysqwdHp6yCNpcXeS84dlprcb-f0z2jF0Y,3160
|
|
3
|
-
koi_net/core.py,sha256=IO8kqiNMYVeuNzilq7eHBA7IulsxRjrCbWnIAx6_abA,4406
|
|
4
|
-
koi_net/identity.py,sha256=muc5vuQ8zUOebhwAB3-ql6W2pgQETiYXXQAFBv8bLyg,1288
|
|
5
|
-
koi_net/network/__init__.py,sha256=r_RN-q_mDYC-2RAkN-lJoMUX76TXyfEUc_MVKW87z0g,39
|
|
6
|
-
koi_net/network/graph.py,sha256=dsfPuHUTkCzlj0QeL0e7dgp7-FR5_AGP7eE8EpBPhC0,4710
|
|
7
|
-
koi_net/network/interface.py,sha256=icpl0rzpC5yV86BQBAbjAjK7AWhgCFoyFLm_Fjf1mCI,10451
|
|
8
|
-
koi_net/network/request_handler.py,sha256=66gjX2x4UnBWZYwKLjp_3WkhL-ekhR3VAyfGviHTcUs,4790
|
|
9
|
-
koi_net/network/response_handler.py,sha256=CAwici2Etj9ESndERXdtYkMlc4gWHz_xc7jHgY2Qjcg,1830
|
|
10
|
-
koi_net/processor/__init__.py,sha256=x4fAY0hvQEDcpfdTB3POIzxBQjYAtn0qQazPo1Xm0m4,41
|
|
11
|
-
koi_net/processor/default_handlers.py,sha256=dP64lEJ64BJ7H8PhFK-GZI1pv51tVVINV4jAgcOtOhc,8669
|
|
12
|
-
koi_net/processor/handler.py,sha256=7X6M6PP8m6-xdtsP1y4QO83g_MN5VSszNNikprITK80,2523
|
|
13
|
-
koi_net/processor/interface.py,sha256=Kyw4SQos_1WdcPJJe-j2w4xDIfwtmpF4mfGlkRVRqUI,12876
|
|
14
|
-
koi_net/processor/knowledge_object.py,sha256=RCgzkILsWm1Jw_NkSu4jTRYA9Ugga6mJ4jqKWwketQs,4090
|
|
15
|
-
koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
koi_net/protocol/api_models.py,sha256=DYDKCRD2Uja633bBAyTsaxyb1oF9pX9yQ9NpNAbkczo,1070
|
|
17
|
-
koi_net/protocol/consts.py,sha256=bisbVEojPIHlLhkLafBzfIhH25TjNfvTORF1g6YXzIM,243
|
|
18
|
-
koi_net/protocol/edge.py,sha256=CcmvIY4P1HEBdKNJ4wFRDmwYMRMss24Besmbi7ZRFxQ,427
|
|
19
|
-
koi_net/protocol/event.py,sha256=eGgihEj1gliLoQRk8pVB2q_was0AGo-PbT3Hqnpn3oU,1379
|
|
20
|
-
koi_net/protocol/helpers.py,sha256=8ZkQrjb_G0QEaMIKe9wkFOBonl1bkmemx_pwKMwIiLg,695
|
|
21
|
-
koi_net/protocol/node.py,sha256=2HhCh3LdBLlY2Z_kXNmKHzpVLKbP_ODob3HjHayFQtM,375
|
|
22
|
-
koi_net-1.0.0b19.dist-info/METADATA,sha256=uVzxDiRiHRncwfjotk7j_frmm_ka6PEpI-kcA278HoE,37107
|
|
23
|
-
koi_net-1.0.0b19.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
24
|
-
koi_net-1.0.0b19.dist-info/licenses/LICENSE,sha256=03mgCL5qth2aD9C3F3qNVs4sFJSpK9kjtYCyOwdSp7s,1069
|
|
25
|
-
koi_net-1.0.0b19.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|