pybgpkitstream 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pybgpkitstream/__init__.py +15 -2
- pybgpkitstream/bgpelement.py +11 -2
- pybgpkitstream/bgpkitstream.py +57 -31
- pybgpkitstream/bgpparser.py +5 -6
- pybgpkitstream/bgpstreamconfig.py +58 -14
- pybgpkitstream/cli.py +0 -3
- pybgpkitstream/rislive.py +141 -0
- pybgpkitstream/utils.py +2 -1
- {pybgpkitstream-0.3.0.dist-info → pybgpkitstream-0.4.0.dist-info}/METADATA +4 -3
- pybgpkitstream-0.4.0.dist-info/RECORD +13 -0
- pybgpkitstream-0.3.0.dist-info/RECORD +0 -12
- {pybgpkitstream-0.3.0.dist-info → pybgpkitstream-0.4.0.dist-info}/WHEEL +0 -0
- {pybgpkitstream-0.3.0.dist-info → pybgpkitstream-0.4.0.dist-info}/entry_points.txt +0 -0
pybgpkitstream/__init__.py
CHANGED
|
@@ -1,4 +1,17 @@
|
|
|
1
|
-
from .bgpstreamconfig import
|
|
1
|
+
from .bgpstreamconfig import (
|
|
2
|
+
BGPStreamConfig,
|
|
3
|
+
FilterOptions,
|
|
4
|
+
PyBGPKITStreamConfig,
|
|
5
|
+
LiveStreamConfig,
|
|
6
|
+
)
|
|
2
7
|
from .bgpkitstream import BGPKITStream
|
|
8
|
+
from .bgpelement import BGPElement
|
|
3
9
|
|
|
4
|
-
__all__ = [
|
|
10
|
+
__all__ = [
|
|
11
|
+
"BGPStreamConfig",
|
|
12
|
+
"FilterOptions",
|
|
13
|
+
"BGPKITStream",
|
|
14
|
+
"PyBGPKITStreamConfig",
|
|
15
|
+
"BGPElement",
|
|
16
|
+
"LiveStreamConfig",
|
|
17
|
+
]
|
pybgpkitstream/bgpelement.py
CHANGED
|
@@ -14,9 +14,9 @@ class ElementFields(TypedDict):
|
|
|
14
14
|
class BGPElement(NamedTuple):
|
|
15
15
|
"""Compatible with pybgpstream.BGPElem"""
|
|
16
16
|
|
|
17
|
+
time: float # time first for sorting tuples convention
|
|
17
18
|
type: str
|
|
18
19
|
collector: str
|
|
19
|
-
time: float
|
|
20
20
|
peer_asn: int
|
|
21
21
|
peer_address: str
|
|
22
22
|
fields: ElementFields
|
|
@@ -31,7 +31,9 @@ class BGPElement(NamedTuple):
|
|
|
31
31
|
self.peer_address,
|
|
32
32
|
self._maybe_field("prefix"),
|
|
33
33
|
self._maybe_field("next-hop"),
|
|
34
|
-
self.
|
|
34
|
+
" ".join(map(str, self.fields["as-path"]))
|
|
35
|
+
if "as-path" in self.fields
|
|
36
|
+
else None,
|
|
35
37
|
" ".join(self.fields["communities"])
|
|
36
38
|
if "communities" in self.fields
|
|
37
39
|
else None,
|
|
@@ -42,3 +44,10 @@ class BGPElement(NamedTuple):
|
|
|
42
44
|
def _maybe_field(self, field):
|
|
43
45
|
"""Credit to pybgpstream"""
|
|
44
46
|
return self.fields[field] if field in self.fields else None
|
|
47
|
+
|
|
48
|
+
# Useful for sorting streams
|
|
49
|
+
def __lt__(self, other):
|
|
50
|
+
return self.time < other.time
|
|
51
|
+
|
|
52
|
+
def __le__(self, other):
|
|
53
|
+
return self.time <= other.time
|
pybgpkitstream/bgpkitstream.py
CHANGED
|
@@ -19,6 +19,7 @@ from pybgpkitstream.bgpstreamconfig import (
|
|
|
19
19
|
BGPStreamConfig,
|
|
20
20
|
FilterOptions,
|
|
21
21
|
PyBGPKITStreamConfig,
|
|
22
|
+
LiveStreamConfig,
|
|
22
23
|
)
|
|
23
24
|
from pybgpkitstream.bgpelement import BGPElement
|
|
24
25
|
from pybgpkitstream.bgpparser import (
|
|
@@ -28,6 +29,7 @@ from pybgpkitstream.bgpparser import (
|
|
|
28
29
|
PyBGPStreamParser,
|
|
29
30
|
BGPdumpParser,
|
|
30
31
|
)
|
|
32
|
+
from pybgpkitstream.rislive import RISLiveStream, jitter_buffer_stream
|
|
31
33
|
from pybgpkitstream.utils import dt_from_filepath
|
|
32
34
|
|
|
33
35
|
name2parser = {
|
|
@@ -41,23 +43,6 @@ name2parser = {
|
|
|
41
43
|
logger = logging.getLogger(__name__)
|
|
42
44
|
|
|
43
45
|
|
|
44
|
-
def convert_bgpkit_elem(element, is_rib: bool, collector: str) -> BGPElement:
|
|
45
|
-
"""Convert pybgpkit element to pybgpstream-like element"""
|
|
46
|
-
return BGPElement(
|
|
47
|
-
type="R" if is_rib else element.elem_type,
|
|
48
|
-
collector=collector,
|
|
49
|
-
time=element.timestamp,
|
|
50
|
-
peer_asn=element.peer_asn,
|
|
51
|
-
peer_address=element.peer_ip,
|
|
52
|
-
fields={
|
|
53
|
-
"next-hop": element.next_hop,
|
|
54
|
-
"as-path": element.as_path,
|
|
55
|
-
"communities": [] if not element.communities else element.communities,
|
|
56
|
-
"prefix": element.prefix,
|
|
57
|
-
},
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
|
|
61
46
|
def crc32(input_str: str):
|
|
62
47
|
input_bytes = input_str.encode("utf-8")
|
|
63
48
|
crc = binascii.crc32(input_bytes) & 0xFFFFFFFF
|
|
@@ -87,21 +72,22 @@ def get_shared_memory():
|
|
|
87
72
|
class BGPKITStream:
|
|
88
73
|
def __init__(
|
|
89
74
|
self,
|
|
90
|
-
|
|
91
|
-
ts_end: float,
|
|
92
|
-
collector_id: str,
|
|
75
|
+
collectors: list[str],
|
|
93
76
|
data_type: list[Literal["update", "rib"]],
|
|
94
|
-
|
|
77
|
+
ts_start: float = None,
|
|
78
|
+
ts_end: float = None,
|
|
79
|
+
filters: FilterOptions | None = None,
|
|
95
80
|
cache_dir: str | None = None,
|
|
96
81
|
max_concurrent_downloads: int | None = 10,
|
|
97
82
|
chunk_time: float | None = datetime.timedelta(hours=2).seconds,
|
|
98
83
|
ram_fetch: bool | None = True,
|
|
99
84
|
parser_name: str | None = "pybgpkit",
|
|
85
|
+
jitter_buffer_delay: float | None = 10.0,
|
|
100
86
|
):
|
|
101
87
|
# Stream config
|
|
102
88
|
self.ts_start = ts_start
|
|
103
89
|
self.ts_end = ts_end
|
|
104
|
-
self.
|
|
90
|
+
self.collectors = collectors
|
|
105
91
|
self.data_type = data_type
|
|
106
92
|
if not filters:
|
|
107
93
|
filters = FilterOptions()
|
|
@@ -126,6 +112,9 @@ class BGPKITStream:
|
|
|
126
112
|
self.broker = bgpkit.Broker()
|
|
127
113
|
self.parser_cls: BGPParser = name2parser[parser_name]
|
|
128
114
|
|
|
115
|
+
# Live config
|
|
116
|
+
self.jitter_buffer_delay = jitter_buffer_delay
|
|
117
|
+
|
|
129
118
|
@staticmethod
|
|
130
119
|
def _generate_cache_filename(url):
|
|
131
120
|
"""Generate a cache filename compatible with BGPKIT parser."""
|
|
@@ -163,7 +152,7 @@ class BGPKITStream:
|
|
|
163
152
|
items: list[BrokerItem] = self.broker.query(
|
|
164
153
|
ts_start=int(self.ts_start - 60),
|
|
165
154
|
ts_end=int(self.ts_end),
|
|
166
|
-
collector_id=self.
|
|
155
|
+
collector_id=",".join(self.collectors),
|
|
167
156
|
data_type=data_type,
|
|
168
157
|
)
|
|
169
158
|
for item in items:
|
|
@@ -226,6 +215,8 @@ class BGPKITStream:
|
|
|
226
215
|
logging.info("All downloads finished.")
|
|
227
216
|
|
|
228
217
|
def __iter__(self):
|
|
218
|
+
if self.ts_start is None and self.ts_end is None:
|
|
219
|
+
return self._iter_live()
|
|
229
220
|
if "update" in self.data_type:
|
|
230
221
|
return self._iter_update()
|
|
231
222
|
else:
|
|
@@ -250,7 +241,7 @@ class BGPKITStream:
|
|
|
250
241
|
ts_start=current,
|
|
251
242
|
ts_end=chunk_end
|
|
252
243
|
- 1, # remove one second because BGPKIT include border
|
|
253
|
-
|
|
244
|
+
collectors=self.collectors,
|
|
254
245
|
data_type=self.data_type,
|
|
255
246
|
cache_dir=self.cache_dir.name
|
|
256
247
|
if isinstance(self.cache_dir, Directory)
|
|
@@ -319,7 +310,7 @@ class BGPKITStream:
|
|
|
319
310
|
ts_start=current,
|
|
320
311
|
ts_end=chunk_end
|
|
321
312
|
- 1, # remove one second because BGPKIT include border
|
|
322
|
-
|
|
313
|
+
collectors=self.collectors,
|
|
323
314
|
data_type=self.data_type,
|
|
324
315
|
cache_dir=self.cache_dir.name
|
|
325
316
|
if isinstance(self.cache_dir, Directory)
|
|
@@ -365,14 +356,30 @@ class BGPKITStream:
|
|
|
365
356
|
finally:
|
|
366
357
|
self.cache_dir.cleanup()
|
|
367
358
|
|
|
359
|
+
def _iter_live(self) -> Iterator[BGPElement]:
|
|
360
|
+
|
|
361
|
+
ris_collectors = [
|
|
362
|
+
collector for collector in self.collectors if collector[:3] == "rrc"
|
|
363
|
+
]
|
|
364
|
+
|
|
365
|
+
stream = RISLiveStream(collectors=ris_collectors, filters=self.filters)
|
|
366
|
+
|
|
367
|
+
if self.jitter_buffer_delay is not None and self.jitter_buffer_delay > 0:
|
|
368
|
+
stream = jitter_buffer_stream(stream, buffer_delay=self.jitter_buffer_delay)
|
|
369
|
+
|
|
370
|
+
for elem in stream:
|
|
371
|
+
yield elem
|
|
372
|
+
|
|
368
373
|
@classmethod
|
|
369
|
-
def from_config(
|
|
374
|
+
def from_config(
|
|
375
|
+
cls, config: PyBGPKITStreamConfig | BGPStreamConfig | LiveStreamConfig
|
|
376
|
+
):
|
|
370
377
|
if isinstance(config, PyBGPKITStreamConfig):
|
|
371
378
|
stream_config = config.bgpstream_config
|
|
372
379
|
return cls(
|
|
373
380
|
ts_start=stream_config.start_time.timestamp(),
|
|
374
381
|
ts_end=stream_config.end_time.timestamp(),
|
|
375
|
-
|
|
382
|
+
collectors=stream_config.collectors,
|
|
376
383
|
data_type=[dtype[:-1] for dtype in stream_config.data_types],
|
|
377
384
|
filters=stream_config.filters
|
|
378
385
|
if stream_config.filters
|
|
@@ -387,10 +394,29 @@ class BGPKITStream:
|
|
|
387
394
|
)
|
|
388
395
|
|
|
389
396
|
elif isinstance(config, BGPStreamConfig):
|
|
397
|
+
if not config.is_live():
|
|
398
|
+
return cls(
|
|
399
|
+
ts_start=config.start_time.timestamp(),
|
|
400
|
+
ts_end=config.end_time.timestamp(),
|
|
401
|
+
collectors=config.collectors,
|
|
402
|
+
data_type=[dtype[:-1] for dtype in config.data_types],
|
|
403
|
+
filters=config.filters if config.filters else FilterOptions(),
|
|
404
|
+
)
|
|
405
|
+
else:
|
|
406
|
+
return cls(
|
|
407
|
+
collectors=config.collectors,
|
|
408
|
+
data_type=["update"],
|
|
409
|
+
filters=config.filters if config.filters else FilterOptions(),
|
|
410
|
+
jitter_buffer_delay=10,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
elif isinstance(config, LiveStreamConfig):
|
|
390
414
|
return cls(
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
collector_id=",".join(config.collectors),
|
|
394
|
-
data_type=[dtype[:-1] for dtype in config.data_types],
|
|
415
|
+
collectors=config.collectors,
|
|
416
|
+
data_type=["update"],
|
|
395
417
|
filters=config.filters if config.filters else FilterOptions(),
|
|
418
|
+
jitter_buffer_delay=config.jitter_buffer_delay,
|
|
396
419
|
)
|
|
420
|
+
|
|
421
|
+
else:
|
|
422
|
+
raise ValueError("Unsupported config type")
|
pybgpkitstream/bgpparser.py
CHANGED
|
@@ -115,9 +115,9 @@ class BGPKITParser(BGPParser):
|
|
|
115
115
|
# Structure: Type|Time|PeerIP|PeerAS|Prefix
|
|
116
116
|
if rec_type == "W":
|
|
117
117
|
return BGPElement(
|
|
118
|
+
time=self.time, # force RIB filename timestamp instead of last changed
|
|
118
119
|
type="W",
|
|
119
120
|
collector=self.collector,
|
|
120
|
-
time=self.time, # force RIB filename timestamp instead of last changed
|
|
121
121
|
peer_asn=int(element[3]),
|
|
122
122
|
peer_address=element[2],
|
|
123
123
|
fields={"prefix": element[4]},
|
|
@@ -133,11 +133,10 @@ class BGPKITParser(BGPParser):
|
|
|
133
133
|
|
|
134
134
|
return BGPElement(
|
|
135
135
|
# bgpkit outputs 'A' for both Updates and RIB entries.
|
|
136
|
-
|
|
136
|
+
self.time,
|
|
137
137
|
"R" if self.is_rib else rec_type,
|
|
138
138
|
self.collector,
|
|
139
139
|
# float(element[1]),
|
|
140
|
-
self.time,
|
|
141
140
|
int(element[3]),
|
|
142
141
|
element[2],
|
|
143
142
|
{
|
|
@@ -242,9 +241,9 @@ class BGPdumpParser(BGPParser):
|
|
|
242
241
|
# 1. Handle Withdrawals (Fastest path, fewer fields)
|
|
243
242
|
if elem_type == "W":
|
|
244
243
|
return BGPElement(
|
|
244
|
+
float(element[1]),
|
|
245
245
|
"W",
|
|
246
246
|
self.collector,
|
|
247
|
-
float(element[1]),
|
|
248
247
|
int(element[4]),
|
|
249
248
|
element[3],
|
|
250
249
|
{"prefix": element[5]}, # Dict literal is faster than assignment
|
|
@@ -257,9 +256,9 @@ class BGPdumpParser(BGPParser):
|
|
|
257
256
|
# Logic: if TABLE_DUMP2, type is R, else A
|
|
258
257
|
# Construct fields dict in one shot (BUILD_MAP opcode)
|
|
259
258
|
return BGPElement(
|
|
259
|
+
float(element[1]),
|
|
260
260
|
"R" if elem_type == "B" else "A",
|
|
261
261
|
self.collector,
|
|
262
|
-
float(element[1]),
|
|
263
262
|
int(element[4]),
|
|
264
263
|
element[3],
|
|
265
264
|
{
|
|
@@ -267,7 +266,7 @@ class BGPdumpParser(BGPParser):
|
|
|
267
266
|
"as-path": element[6],
|
|
268
267
|
"next-hop": element[8],
|
|
269
268
|
# Check for empty string before splitting (avoids creating [''])
|
|
270
|
-
"communities": rec_comm.split() if rec_comm else [],
|
|
269
|
+
"communities": rec_comm.split(" ") if rec_comm else [],
|
|
271
270
|
},
|
|
272
271
|
)
|
|
273
272
|
|
|
@@ -48,20 +48,27 @@ class FilterOptions(BaseModel):
|
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
class BGPStreamConfig(BaseModel):
|
|
51
|
-
"""Unified BGPStream config
|
|
51
|
+
"""Unified BGPStream config"""
|
|
52
52
|
|
|
53
|
-
start_time: datetime.datetime = Field(
|
|
54
|
-
|
|
53
|
+
start_time: datetime.datetime | None = Field(
|
|
54
|
+
default=None, description="Start of the stream"
|
|
55
|
+
)
|
|
56
|
+
end_time: datetime.datetime | None = Field(
|
|
57
|
+
default=None, description="End of the stream"
|
|
58
|
+
)
|
|
55
59
|
collectors: list[str] = Field(description="List of collectors to get data from")
|
|
56
|
-
data_types: list[Literal["ribs", "updates"]] = Field(
|
|
57
|
-
|
|
60
|
+
data_types: list[Literal["ribs", "updates"]] | None = Field(
|
|
61
|
+
default=["updates"],
|
|
62
|
+
description="List of archives files to consider (`ribs` or `updates`)",
|
|
58
63
|
)
|
|
59
64
|
|
|
60
65
|
filters: FilterOptions | None = Field(default=None, description="Optional filters")
|
|
61
66
|
|
|
62
|
-
@field_validator("start_time", "end_time")
|
|
67
|
+
@field_validator("start_time", "end_time", mode="before")
|
|
63
68
|
@classmethod
|
|
64
69
|
def normalize_to_utc(cls, dt: datetime.datetime) -> datetime.datetime:
|
|
70
|
+
if dt is None:
|
|
71
|
+
return None
|
|
65
72
|
# if naive datetime (not timezone-aware) assume it's UTC
|
|
66
73
|
if dt.tzinfo is None:
|
|
67
74
|
return dt.replace(tzinfo=datetime.timezone.utc)
|
|
@@ -69,6 +76,38 @@ class BGPStreamConfig(BaseModel):
|
|
|
69
76
|
else:
|
|
70
77
|
return dt.astimezone(datetime.timezone.utc)
|
|
71
78
|
|
|
79
|
+
@model_validator(mode="after")
|
|
80
|
+
def validate(self) -> "BGPStreamConfig":
|
|
81
|
+
|
|
82
|
+
if (self.start_time is None) ^ (self.end_time is None):
|
|
83
|
+
raise ValueError(
|
|
84
|
+
"Provide both start and end times, or leave both as None for live mode."
|
|
85
|
+
)
|
|
86
|
+
if not self.is_live():
|
|
87
|
+
assert self.start_time < self.end_time
|
|
88
|
+
# Force data_type to update for live mode
|
|
89
|
+
else:
|
|
90
|
+
if self.data_types is None:
|
|
91
|
+
self.data_types = ["updates"]
|
|
92
|
+
|
|
93
|
+
return self
|
|
94
|
+
|
|
95
|
+
def is_live(self) -> bool:
|
|
96
|
+
return self.start_time is None and self.end_time is None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class LiveStreamConfig(BaseModel):
|
|
100
|
+
"""Config for live mode"""
|
|
101
|
+
|
|
102
|
+
collectors: list[str] = Field(
|
|
103
|
+
description="List of collectors to get data from (for now only RIS live collectors)"
|
|
104
|
+
)
|
|
105
|
+
filters: FilterOptions | None = Field(default=None, description="Optional filters")
|
|
106
|
+
jitter_buffer_delay: float | None = Field(
|
|
107
|
+
default=10.0,
|
|
108
|
+
description="Jitter buffer time in seconds to make sure RIS live updates are time-sorted. Introduce a slight delay. Set to None or 0 to disable",
|
|
109
|
+
)
|
|
110
|
+
|
|
72
111
|
|
|
73
112
|
class PyBGPKITStreamConfig(BaseModel):
|
|
74
113
|
"""Unified BGPStream config and parameters related to PyBGPKIT implementation (all optional)"""
|
|
@@ -135,26 +174,31 @@ class PyBGPKITStreamConfig(BaseModel):
|
|
|
135
174
|
raise ValueError(
|
|
136
175
|
"bgpkit binary not found in PATH. "
|
|
137
176
|
"Install from: https://github.com/bgpkit/bgpkit-parser "
|
|
138
|
-
"or use cargo: cargo install bgpkit-parser"
|
|
177
|
+
"or use cargo: cargo install bgpkit-parser --features cli"
|
|
139
178
|
)
|
|
140
179
|
|
|
141
|
-
# Return the parser value if validation passes
|
|
142
180
|
return parser
|
|
143
|
-
|
|
144
|
-
@model_validator(mode=
|
|
181
|
+
|
|
182
|
+
@model_validator(mode="before")
|
|
145
183
|
@classmethod
|
|
146
184
|
def nest_bgpstream_params(cls, data: dict) -> dict:
|
|
147
185
|
"""Allow to define a flat config"""
|
|
148
186
|
# If the user already provided 'bgpstream_config', do nothing
|
|
149
187
|
if "bgpstream_config" in data:
|
|
150
188
|
return data
|
|
151
|
-
|
|
189
|
+
|
|
152
190
|
# Define which fields belong to the inner BGPStreamConfig
|
|
153
|
-
stream_fields = {
|
|
154
|
-
|
|
191
|
+
stream_fields = {
|
|
192
|
+
"start_time",
|
|
193
|
+
"end_time",
|
|
194
|
+
"collectors",
|
|
195
|
+
"data_types",
|
|
196
|
+
"filters",
|
|
197
|
+
}
|
|
198
|
+
|
|
155
199
|
# Extract those fields from the flat input
|
|
156
200
|
inner_data = {k: data.pop(k) for k in stream_fields if k in data}
|
|
157
|
-
|
|
201
|
+
|
|
158
202
|
# Nest them back into the dictionary
|
|
159
203
|
data["bgpstream_config"] = inner_data
|
|
160
204
|
return data
|
pybgpkitstream/cli.py
CHANGED
|
@@ -154,13 +154,10 @@ def main():
|
|
|
154
154
|
bgpstream_config=bgpstream_config, cache_dir=args.cache_dir, parser=args.parser
|
|
155
155
|
)
|
|
156
156
|
|
|
157
|
-
for element in BGPKITStream.from_config(config):
|
|
158
|
-
print(element)
|
|
159
157
|
try:
|
|
160
158
|
for element in BGPKITStream.from_config(config):
|
|
161
159
|
print(element)
|
|
162
160
|
except Exception as e:
|
|
163
|
-
print(e)
|
|
164
161
|
print(f"An error occurred during streaming: {e}", file=sys.stderr)
|
|
165
162
|
sys.exit(1)
|
|
166
163
|
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
from typing import Iterator
|
|
2
|
+
import json
|
|
3
|
+
import heapq
|
|
4
|
+
import websocket
|
|
5
|
+
|
|
6
|
+
from pybgpkitstream.bgpelement import BGPElement
|
|
7
|
+
from pybgpkitstream.bgpstreamconfig import FilterOptions
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def ris_message2bgpelem(ris_message: dict) -> Iterator[BGPElement]:
|
|
11
|
+
|
|
12
|
+
timestamp = float(ris_message["timestamp"])
|
|
13
|
+
collector = ris_message["host"].split(".")[0]
|
|
14
|
+
peer_asn = int(ris_message["peer_asn"])
|
|
15
|
+
peer_address = ris_message["peer"]
|
|
16
|
+
path = ris_message["path"]
|
|
17
|
+
communities = ris_message["community"]
|
|
18
|
+
if communities:
|
|
19
|
+
communities = [f"{asn}:{community}" for asn, community in communities]
|
|
20
|
+
|
|
21
|
+
for pfx in ris_message["withdrawals"]:
|
|
22
|
+
yield BGPElement(
|
|
23
|
+
type="W",
|
|
24
|
+
collector=collector,
|
|
25
|
+
time=timestamp,
|
|
26
|
+
peer_asn=peer_asn,
|
|
27
|
+
peer_address=peer_address,
|
|
28
|
+
fields={
|
|
29
|
+
"as-path": path,
|
|
30
|
+
"communities": communities,
|
|
31
|
+
"prefix": pfx,
|
|
32
|
+
},
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
for announcement in ris_message["announcements"]:
|
|
36
|
+
for pfx in announcement["prefixes"]:
|
|
37
|
+
yield BGPElement(
|
|
38
|
+
type="A",
|
|
39
|
+
collector=collector,
|
|
40
|
+
time=timestamp,
|
|
41
|
+
peer_asn=peer_asn,
|
|
42
|
+
peer_address=peer_address,
|
|
43
|
+
fields={
|
|
44
|
+
"next-hop": announcement["next_hop"].split(",")[0],
|
|
45
|
+
"as-path": path,
|
|
46
|
+
"communities": communities,
|
|
47
|
+
"prefix": pfx,
|
|
48
|
+
},
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class RISLiveStream:
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
collectors: list[str],
|
|
56
|
+
client="pybgpkitstream",
|
|
57
|
+
filters: FilterOptions = None,
|
|
58
|
+
):
|
|
59
|
+
self.collectors = collectors
|
|
60
|
+
self.client = client
|
|
61
|
+
print(filters)
|
|
62
|
+
self.filters = self._convert_filter_options(filters)
|
|
63
|
+
print(self.filters)
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def _convert_filter_options(f: FilterOptions) -> dict:
|
|
67
|
+
"""Convert FilterOptions to RIS live filters"""
|
|
68
|
+
if f is None:
|
|
69
|
+
return {}
|
|
70
|
+
|
|
71
|
+
if not f.model_dump(exclude_unset=True):
|
|
72
|
+
return {}
|
|
73
|
+
|
|
74
|
+
res = {}
|
|
75
|
+
if f.update_type == "withdraw":
|
|
76
|
+
res["require"] = "withdrawals"
|
|
77
|
+
elif f.update_type == "announce":
|
|
78
|
+
res["require"] = "announcements"
|
|
79
|
+
if f.peer_ip:
|
|
80
|
+
res["peer"] = f.peer_ip
|
|
81
|
+
path_elements = []
|
|
82
|
+
if f.peer_asn:
|
|
83
|
+
path_elements.append(f"^{f.peer_asn}")
|
|
84
|
+
if f.origin_asn:
|
|
85
|
+
path_elements.append(f"{f.origin_asn}$")
|
|
86
|
+
res["path"] = ",".join(path_elements)
|
|
87
|
+
|
|
88
|
+
if f.prefix:
|
|
89
|
+
res["prefix"] = f.prefix
|
|
90
|
+
# default is True which I think is not consistent with BGPKIT/BGPStream
|
|
91
|
+
res["moreSpecific"] = False
|
|
92
|
+
if f.prefix_sub:
|
|
93
|
+
res["prefix"] = f.prefix_sub
|
|
94
|
+
res["moreSpecific"] = True
|
|
95
|
+
if f.prefix_super:
|
|
96
|
+
res["prefix"] = f.prefix_super
|
|
97
|
+
res["lessSpecific"] = True
|
|
98
|
+
if f.prefix_super_sub:
|
|
99
|
+
res["prefix"] = f.prefix_super_sub
|
|
100
|
+
res["moreSpecific"] = True
|
|
101
|
+
res["lessSpecific"] = True
|
|
102
|
+
|
|
103
|
+
return res
|
|
104
|
+
|
|
105
|
+
def __iter__(self) -> Iterator[BGPElement]:
|
|
106
|
+
ws = websocket.WebSocket()
|
|
107
|
+
ws.connect(f"wss://ris-live.ripe.net/v1/ws/?client={self.client}")
|
|
108
|
+
|
|
109
|
+
# Subscribe to each collector on the same connection
|
|
110
|
+
for collector in self.collectors:
|
|
111
|
+
params = {"host": collector, "type": "UPDATE"}
|
|
112
|
+
params = params | self.filters
|
|
113
|
+
print(params)
|
|
114
|
+
ws.send(json.dumps({"type": "ris_subscribe", "data": params}))
|
|
115
|
+
|
|
116
|
+
for data in ws:
|
|
117
|
+
parsed = json.loads(data)["data"]
|
|
118
|
+
yield from ris_message2bgpelem(parsed)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def jitter_buffer_stream(stream, buffer_delay=10) -> Iterator[BGPElement]:
|
|
122
|
+
"""
|
|
123
|
+
Produces an ordered stream by buffering elements for `buffer_delay` seconds.
|
|
124
|
+
"""
|
|
125
|
+
heap = []
|
|
126
|
+
max_ts_seen = float("-inf")
|
|
127
|
+
|
|
128
|
+
for elem in stream:
|
|
129
|
+
# Track the latest timestamp seen in the jittery stream
|
|
130
|
+
if elem.time > max_ts_seen:
|
|
131
|
+
max_ts_seen = elem.time
|
|
132
|
+
|
|
133
|
+
heapq.heappush(heap, elem)
|
|
134
|
+
|
|
135
|
+
# Flush from buffer if timestamp is old enough
|
|
136
|
+
while heap and (max_ts_seen - heap[0].time) > buffer_delay:
|
|
137
|
+
yield heapq.heappop(heap)
|
|
138
|
+
|
|
139
|
+
# Clean up when stream ends (never hopefully)
|
|
140
|
+
while heap:
|
|
141
|
+
yield heapq.heappop(heap)
|
pybgpkitstream/utils.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import re
|
|
3
3
|
|
|
4
|
+
|
|
4
5
|
def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datetime:
|
|
5
6
|
match = re.search(pattern, filepath)
|
|
6
7
|
if not match:
|
|
@@ -8,4 +9,4 @@ def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datet
|
|
|
8
9
|
timestamp_str = match.group(1)
|
|
9
10
|
dt = datetime.datetime.strptime(timestamp_str, "%Y%m%d.%H%M")
|
|
10
11
|
dt = dt.replace(tzinfo=datetime.timezone.utc)
|
|
11
|
-
return dt
|
|
12
|
+
return dt
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pybgpkitstream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Drop-in replacement for PyBGPStream using BGPKIT
|
|
5
5
|
Author: JustinLoye
|
|
6
6
|
Author-email: JustinLoye <jloye@iij.ad.jp>
|
|
7
7
|
Requires-Dist: aiohttp>=3.12.15
|
|
8
8
|
Requires-Dist: pybgpkit>=0.6.2
|
|
9
9
|
Requires-Dist: pydantic>=2.11.9
|
|
10
|
+
Requires-Dist: websocket-client>=1.8.0
|
|
10
11
|
Requires-Python: >=3.10
|
|
11
12
|
Description-Content-Type: text/markdown
|
|
12
13
|
|
|
@@ -67,5 +68,5 @@ BGPKIT broker and parser are great, but cannot be used to create an ordered stre
|
|
|
67
68
|
|
|
68
69
|
## Missing features
|
|
69
70
|
|
|
70
|
-
-
|
|
71
|
-
-
|
|
71
|
+
- Live mode for RouteViews collectors
|
|
72
|
+
- Some PyBGPStream data interface options like csv or sqlite
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
pybgpkitstream/__init__.py,sha256=_i1D2bx1SanDe8BZ8B4S1349an7-1JW4ptsY5T8gig4,344
|
|
2
|
+
pybgpkitstream/bgpelement.py,sha256=Kr4YLk14vRZtQeYQv_k-NcODmgP0MHdXaIlUzeRERjk,1466
|
|
3
|
+
pybgpkitstream/bgpkitstream.py,sha256=Bw7V4RYg7Bmkkl2XG4LPiFZGEECOkijcd59EUxsBBbQ,16031
|
|
4
|
+
pybgpkitstream/bgpparser.py,sha256=uQJmbaRQOXsCkTnTW8Lhi336cLrcMWCaoAyNe4ZE1B0,16177
|
|
5
|
+
pybgpkitstream/bgpstreamconfig.py,sha256=PaXMREUFb6zj9Y4Zpnlb1nekV8zGYIyEtvSp99hDuic,7480
|
|
6
|
+
pybgpkitstream/cli.py,sha256=U1jFjEwkuySk7OhUR2sYWkiYFEx2XDLGsSwZM70zGcE,4548
|
|
7
|
+
pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pybgpkitstream/rislive.py,sha256=e0o8qF6nT_hJnDbU7f8ZuPdTwEnO0H3lHlJMqp3rqlU,4437
|
|
9
|
+
pybgpkitstream/utils.py,sha256=BzPUKRdJ48bP73eT4LWByrmN0IIIUdBYuATINMXWpTE,406
|
|
10
|
+
pybgpkitstream-0.4.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
|
|
11
|
+
pybgpkitstream-0.4.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
|
|
12
|
+
pybgpkitstream-0.4.0.dist-info/METADATA,sha256=F4sExUM0TL01r8MmV6bftKQWrkH93i18TyreJDwuIuM,2320
|
|
13
|
+
pybgpkitstream-0.4.0.dist-info/RECORD,,
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
pybgpkitstream/__init__.py,sha256=OGWVhZdSvialNkIkQ1VBrmiyOcwkCA1D5IaLo7WQnPI,209
|
|
2
|
-
pybgpkitstream/bgpelement.py,sha256=7mXSUmWThhIbKy2JVsLchoteve0BshT3uH8cdbAe0Go,1176
|
|
3
|
-
pybgpkitstream/bgpkitstream.py,sha256=CKQv7dU-ooznuD1AjHKnZ6qRdPH1ZiOIEGtVNtU8PCY,15062
|
|
4
|
-
pybgpkitstream/bgpparser.py,sha256=eDTWV6iGZTxgF7m78UmBGLDDY6lGOc9TWsfSdJLhiY8,16264
|
|
5
|
-
pybgpkitstream/bgpstreamconfig.py,sha256=vIfEN475WDIZ7kGmi3dnj_1GIQE_r6qkDFETZmMvH5E,6199
|
|
6
|
-
pybgpkitstream/cli.py,sha256=4F5yCNW6OcQFJAj2Hp0rBrDmUDkR1O9x-_aKhVpXrL4,4641
|
|
7
|
-
pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
pybgpkitstream/utils.py,sha256=6FwEEpBtY_20BDlJPOPFmTYQGqw7fCBLjXmnd7gjBdQ,404
|
|
9
|
-
pybgpkitstream-0.3.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
|
|
10
|
-
pybgpkitstream-0.3.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
|
|
11
|
-
pybgpkitstream-0.3.0.dist-info/METADATA,sha256=kzkTcOUY8tHeZxIqbCo4pRrfQ3z3ZIzWvAbNZ5ULlfM,2356
|
|
12
|
-
pybgpkitstream-0.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|