pybgpkitstream 0.3.0__tar.gz → 0.4.0__tar.gz

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.3
 Name: pybgpkitstream
-Version: 0.3.0
+Version: 0.4.0
 Summary: Drop-in replacement for PyBGPStream using BGPKIT
 Author: JustinLoye
 Author-email: JustinLoye <jloye@iij.ad.jp>
 Requires-Dist: aiohttp>=3.12.15
 Requires-Dist: pybgpkit>=0.6.2
 Requires-Dist: pydantic>=2.11.9
+Requires-Dist: websocket-client>=1.8.0
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 
@@ -67,5 +68,5 @@ BGPKIT broker and parser are great, but cannot be used to create an ordered stre
 
 ## Missing features
 
-- live mode (I plan to add semi-live soon.)
-- `pybgpkitstream.BGPElement` is not fully compatible with `pybgpstream.BGPElem`: missing record_type, project, router, router_ip
+- Live mode for RouteViews collectors
+- Some PyBGPStream data interface options like csv or sqlite

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/README.md

@@ -55,5 +55,5 @@ BGPKIT broker and parser are great, but cannot be used to create an ordered stre
 
 ## Missing features
 
-- live mode (I plan to add semi-live soon.)
-- `pybgpkitstream.BGPElement` is not fully compatible with `pybgpstream.BGPElem`: missing record_type, project, router, router_ip
+- Live mode for RouteViews collectors
+- Some PyBGPStream data interface options like csv or sqlite

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pybgpkitstream"
-version = "0.3.0"
+version = "0.4.0"
 description = "Drop-in replacement for PyBGPStream using BGPKIT"
 readme = "README.md"
 authors = [
@@ -11,6 +11,7 @@ dependencies = [
     "aiohttp>=3.12.15",
     "pybgpkit>=0.6.2",
     "pydantic>=2.11.9",
+    "websocket-client>=1.8.0",
 ]
 
 [build-system]

pybgpkitstream-0.4.0/src/pybgpkitstream/__init__.py

@@ -0,0 +1,17 @@
+from .bgpstreamconfig import (
+    BGPStreamConfig,
+    FilterOptions,
+    PyBGPKITStreamConfig,
+    LiveStreamConfig,
+)
+from .bgpkitstream import BGPKITStream
+from .bgpelement import BGPElement
+
+__all__ = [
+    "BGPStreamConfig",
+    "FilterOptions",
+    "BGPKITStream",
+    "PyBGPKITStreamConfig",
+    "BGPElement",
+    "LiveStreamConfig",
+]

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/bgpelement.py

@@ -14,9 +14,9 @@ class ElementFields(TypedDict):
 class BGPElement(NamedTuple):
     """Compatible with pybgpstream.BGPElem"""
 
+    time: float  # time first for sorting tuples convention
     type: str
     collector: str
-    time: float
     peer_asn: int
     peer_address: str
     fields: ElementFields
@@ -31,7 +31,9 @@ class BGPElement(NamedTuple):
             self.peer_address,
             self._maybe_field("prefix"),
             self._maybe_field("next-hop"),
-            self._maybe_field("as-path"),
+            " ".join(map(str, self.fields["as-path"]))
+            if "as-path" in self.fields
+            else None,
             " ".join(self.fields["communities"])
             if "communities" in self.fields
             else None,
@@ -42,3 +44,10 @@ class BGPElement(NamedTuple):
     def _maybe_field(self, field):
         """Credit to pybgpstream"""
         return self.fields[field] if field in self.fields else None
+
+    # Useful for sorting streams
+    def __lt__(self, other):
+        return self.time < other.time
+
+    def __le__(self, other):
+        return self.time <= other.time

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/bgpkitstream.py

@@ -19,6 +19,7 @@ from pybgpkitstream.bgpstreamconfig import (
     BGPStreamConfig,
     FilterOptions,
     PyBGPKITStreamConfig,
+    LiveStreamConfig,
 )
 from pybgpkitstream.bgpelement import BGPElement
 from pybgpkitstream.bgpparser import (
@@ -28,6 +29,7 @@ from pybgpkitstream.bgpparser import (
     PyBGPStreamParser,
     BGPdumpParser,
 )
+from pybgpkitstream.rislive import RISLiveStream, jitter_buffer_stream
 from pybgpkitstream.utils import dt_from_filepath
 
 name2parser = {
@@ -41,23 +43,6 @@ name2parser = {
 logger = logging.getLogger(__name__)
 
 
-def convert_bgpkit_elem(element, is_rib: bool, collector: str) -> BGPElement:
-    """Convert pybgpkit element to pybgpstream-like element"""
-    return BGPElement(
-        type="R" if is_rib else element.elem_type,
-        collector=collector,
-        time=element.timestamp,
-        peer_asn=element.peer_asn,
-        peer_address=element.peer_ip,
-        fields={
-            "next-hop": element.next_hop,
-            "as-path": element.as_path,
-            "communities": [] if not element.communities else element.communities,
-            "prefix": element.prefix,
-        },
-    )
-
-
 def crc32(input_str: str):
     input_bytes = input_str.encode("utf-8")
     crc = binascii.crc32(input_bytes) & 0xFFFFFFFF
@@ -87,21 +72,22 @@ def get_shared_memory():
 class BGPKITStream:
     def __init__(
         self,
-        ts_start: float,
-        ts_end: float,
-        collector_id: str,
+        collectors: list[str],
         data_type: list[Literal["update", "rib"]],
-        filters: FilterOptions | None,
+        ts_start: float = None,
+        ts_end: float = None,
+        filters: FilterOptions | None = None,
         cache_dir: str | None = None,
         max_concurrent_downloads: int | None = 10,
         chunk_time: float | None = datetime.timedelta(hours=2).seconds,
         ram_fetch: bool | None = True,
         parser_name: str | None = "pybgpkit",
+        jitter_buffer_delay: float | None = 10.0,
     ):
         # Stream config
         self.ts_start = ts_start
         self.ts_end = ts_end
-        self.collector_id = collector_id
+        self.collectors = collectors
         self.data_type = data_type
         if not filters:
             filters = FilterOptions()
@@ -126,6 +112,9 @@ class BGPKITStream:
         self.broker = bgpkit.Broker()
         self.parser_cls: BGPParser = name2parser[parser_name]
 
+        # Live config
+        self.jitter_buffer_delay = jitter_buffer_delay
+
     @staticmethod
     def _generate_cache_filename(url):
         """Generate a cache filename compatible with BGPKIT parser."""
@@ -163,7 +152,7 @@ class BGPKITStream:
             items: list[BrokerItem] = self.broker.query(
                 ts_start=int(self.ts_start - 60),
                 ts_end=int(self.ts_end),
-                collector_id=self.collector_id,
+                collector_id=",".join(self.collectors),
                 data_type=data_type,
             )
             for item in items:
@@ -226,6 +215,8 @@ class BGPKITStream:
                 logging.info("All downloads finished.")
 
     def __iter__(self):
+        if self.ts_start is None and self.ts_end is None:
+            return self._iter_live()
         if "update" in self.data_type:
             return self._iter_update()
         else:
@@ -250,7 +241,7 @@ class BGPKITStream:
                         ts_start=current,
                         ts_end=chunk_end
                         - 1,  # remove one second because BGPKIT include border
-                        collector_id=self.collector_id,
+                        collectors=self.collectors,
                         data_type=self.data_type,
                         cache_dir=self.cache_dir.name
                         if isinstance(self.cache_dir, Directory)
@@ -319,7 +310,7 @@ class BGPKITStream:
                         ts_start=current,
                         ts_end=chunk_end
                         - 1,  # remove one second because BGPKIT include border
-                        collector_id=self.collector_id,
+                        collectors=self.collectors,
                         data_type=self.data_type,
                         cache_dir=self.cache_dir.name
                         if isinstance(self.cache_dir, Directory)
@@ -365,14 +356,30 @@ class BGPKITStream:
         finally:
             self.cache_dir.cleanup()
 
+    def _iter_live(self) -> Iterator[BGPElement]:
+
+        ris_collectors = [
+            collector for collector in self.collectors if collector[:3] == "rrc"
+        ]
+
+        stream = RISLiveStream(collectors=ris_collectors, filters=self.filters)
+
+        if self.jitter_buffer_delay is not None and self.jitter_buffer_delay > 0:
+            stream = jitter_buffer_stream(stream, buffer_delay=self.jitter_buffer_delay)
+
+        for elem in stream:
+            yield elem
+
     @classmethod
-    def from_config(cls, config: PyBGPKITStreamConfig | BGPStreamConfig):
+    def from_config(
+        cls, config: PyBGPKITStreamConfig | BGPStreamConfig | LiveStreamConfig
+    ):
         if isinstance(config, PyBGPKITStreamConfig):
             stream_config = config.bgpstream_config
             return cls(
                 ts_start=stream_config.start_time.timestamp(),
                 ts_end=stream_config.end_time.timestamp(),
-                collector_id=",".join(stream_config.collectors),
+                collectors=stream_config.collectors,
                 data_type=[dtype[:-1] for dtype in stream_config.data_types],
                 filters=stream_config.filters
                 if stream_config.filters
@@ -387,10 +394,29 @@ class BGPKITStream:
             )
 
         elif isinstance(config, BGPStreamConfig):
+            if not config.is_live():
+                return cls(
+                    ts_start=config.start_time.timestamp(),
+                    ts_end=config.end_time.timestamp(),
+                    collectors=config.collectors,
+                    data_type=[dtype[:-1] for dtype in config.data_types],
+                    filters=config.filters if config.filters else FilterOptions(),
+                )
+            else:
+                return cls(
+                    collectors=config.collectors,
+                    data_type=["update"],
+                    filters=config.filters if config.filters else FilterOptions(),
+                    jitter_buffer_delay=10,
+                )
+
+        elif isinstance(config, LiveStreamConfig):
             return cls(
-                ts_start=config.start_time.timestamp(),
-                ts_end=config.end_time.timestamp(),
-                collector_id=",".join(config.collectors),
-                data_type=[dtype[:-1] for dtype in config.data_types],
+                collectors=config.collectors,
+                data_type=["update"],
                 filters=config.filters if config.filters else FilterOptions(),
+                jitter_buffer_delay=config.jitter_buffer_delay,
             )
+
+        else:
+            raise ValueError("Unsupported config type")

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/bgpparser.py

@@ -115,9 +115,9 @@ class BGPKITParser(BGPParser):
         # Structure: Type|Time|PeerIP|PeerAS|Prefix
         if rec_type == "W":
             return BGPElement(
+                time=self.time,  # force RIB filename timestamp instead of last changed
                 type="W",
                 collector=self.collector,
-                time=self.time,  # force RIB filename timestamp instead of last changed
                 peer_asn=int(element[3]),
                 peer_address=element[2],
                 fields={"prefix": element[4]},
@@ -133,11 +133,10 @@ class BGPKITParser(BGPParser):
 
         return BGPElement(
             # bgpkit outputs 'A' for both Updates and RIB entries.
-            # Map to "A" (Announcement) or change to "R" if you strictly need RIB typing.
+            self.time,
             "R" if self.is_rib else rec_type,
             self.collector,
             # float(element[1]),
-            self.time,
             int(element[3]),
             element[2],
             {
@@ -242,9 +241,9 @@ class BGPdumpParser(BGPParser):
         # 1. Handle Withdrawals (Fastest path, fewer fields)
         if elem_type == "W":
             return BGPElement(
+                float(element[1]),
                 "W",
                 self.collector,
-                float(element[1]),
                 int(element[4]),
                 element[3],
                 {"prefix": element[5]},  # Dict literal is faster than assignment
@@ -257,9 +256,9 @@ class BGPdumpParser(BGPParser):
         # Logic: if TABLE_DUMP2, type is R, else A
         # Construct fields dict in one shot (BUILD_MAP opcode)
         return BGPElement(
+            float(element[1]),
             "R" if elem_type == "B" else "A",
             self.collector,
-            float(element[1]),
             int(element[4]),
             element[3],
             {
@@ -267,7 +266,7 @@ class BGPdumpParser(BGPParser):
                 "as-path": element[6],
                 "next-hop": element[8],
                 # Check for empty string before splitting (avoids creating [''])
-                "communities": rec_comm.split() if rec_comm else [],
+                "communities": rec_comm.split(" ") if rec_comm else [],
             },
         )
 

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/bgpstreamconfig.py

@@ -48,20 +48,27 @@ class FilterOptions(BaseModel):
 
 
 class BGPStreamConfig(BaseModel):
-    """Unified BGPStream config, compatible with BGPKIT and pybgpstream"""
+    """Unified BGPStream config"""
 
-    start_time: datetime.datetime = Field(description="Start of the stream")
-    end_time: datetime.datetime = Field(description="End of the stream")
+    start_time: datetime.datetime | None = Field(
+        default=None, description="Start of the stream"
+    )
+    end_time: datetime.datetime | None = Field(
+        default=None, description="End of the stream"
+    )
     collectors: list[str] = Field(description="List of collectors to get data from")
-    data_types: list[Literal["ribs", "updates"]] = Field(
-        description="List of archives files to consider (`ribs` or `updates`)"
+    data_types: list[Literal["ribs", "updates"]] | None = Field(
+        default=["updates"],
+        description="List of archives files to consider (`ribs` or `updates`)",
     )
 
     filters: FilterOptions | None = Field(default=None, description="Optional filters")
 
-    @field_validator("start_time", "end_time")
+    @field_validator("start_time", "end_time", mode="before")
     @classmethod
     def normalize_to_utc(cls, dt: datetime.datetime) -> datetime.datetime:
+        if dt is None:
+            return None
         # if naive datetime (not timezone-aware) assume it's UTC
         if dt.tzinfo is None:
             return dt.replace(tzinfo=datetime.timezone.utc)
@@ -69,6 +76,38 @@ class BGPStreamConfig(BaseModel):
         else:
             return dt.astimezone(datetime.timezone.utc)
 
+    @model_validator(mode="after")
+    def validate(self) -> "BGPStreamConfig":
+
+        if (self.start_time is None) ^ (self.end_time is None):
+            raise ValueError(
+                "Provide both start and end times, or leave both as None for live mode."
+            )
+        if not self.is_live():
+            assert self.start_time < self.end_time
+        # Force data_type to update for live mode
+        else:
+            if self.data_types is None:
+                self.data_types = ["updates"]
+
+        return self
+
+    def is_live(self) -> bool:
+        return self.start_time is None and self.end_time is None
+
+
+class LiveStreamConfig(BaseModel):
+    """Config for live mode"""
+
+    collectors: list[str] = Field(
+        description="List of collectors to get data from (for now only RIS live collectors)"
+    )
+    filters: FilterOptions | None = Field(default=None, description="Optional filters")
+    jitter_buffer_delay: float | None = Field(
+        default=10.0,
+        description="Jitter buffer time in seconds to make sure RIS live updates are time-sorted. Introduce a slight delay. Set to None or 0 to disable",
+    )
+
 
 class PyBGPKITStreamConfig(BaseModel):
     """Unified BGPStream config and parameters related to PyBGPKIT implementation (all optional)"""
@@ -135,26 +174,31 @@ class PyBGPKITStreamConfig(BaseModel):
                 raise ValueError(
                     "bgpkit binary not found in PATH. "
                     "Install from: https://github.com/bgpkit/bgpkit-parser "
-                    "or use cargo: cargo install bgpkit-parser"
+                    "or use cargo: cargo install bgpkit-parser --features cli"
                 )
 
-        # Return the parser value if validation passes
         return parser
-    
-    @model_validator(mode='before')
+
+    @model_validator(mode="before")
     @classmethod
     def nest_bgpstream_params(cls, data: dict) -> dict:
         """Allow to define a flat config"""
         # If the user already provided 'bgpstream_config', do nothing
         if "bgpstream_config" in data:
             return data
-        
+
         # Define which fields belong to the inner BGPStreamConfig
-        stream_fields = {"start_time", "end_time", "collectors", "data_types", "filters"}
-        
+        stream_fields = {
+            "start_time",
+            "end_time",
+            "collectors",
+            "data_types",
+            "filters",
+        }
+
         # Extract those fields from the flat input
         inner_data = {k: data.pop(k) for k in stream_fields if k in data}
-        
+
         # Nest them back into the dictionary
         data["bgpstream_config"] = inner_data
         return data

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/cli.py

@@ -154,13 +154,10 @@ def main():
         bgpstream_config=bgpstream_config, cache_dir=args.cache_dir, parser=args.parser
     )
 
-    for element in BGPKITStream.from_config(config):
-        print(element)
     try:
         for element in BGPKITStream.from_config(config):
             print(element)
     except Exception as e:
-        print(e)
         print(f"An error occurred during streaming: {e}", file=sys.stderr)
         sys.exit(1)
 

pybgpkitstream-0.4.0/src/pybgpkitstream/rislive.py

@@ -0,0 +1,141 @@
+from typing import Iterator
+import json
+import heapq
+import websocket
+
+from pybgpkitstream.bgpelement import BGPElement
+from pybgpkitstream.bgpstreamconfig import FilterOptions
+
+
+def ris_message2bgpelem(ris_message: dict) -> Iterator[BGPElement]:
+
+    timestamp = float(ris_message["timestamp"])
+    collector = ris_message["host"].split(".")[0]
+    peer_asn = int(ris_message["peer_asn"])
+    peer_address = ris_message["peer"]
+    path = ris_message["path"]
+    communities = ris_message["community"]
+    if communities:
+        communities = [f"{asn}:{community}" for asn, community in communities]
+
+    for pfx in ris_message["withdrawals"]:
+        yield BGPElement(
+            type="W",
+            collector=collector,
+            time=timestamp,
+            peer_asn=peer_asn,
+            peer_address=peer_address,
+            fields={
+                "as-path": path,
+                "communities": communities,
+                "prefix": pfx,
+            },
+        )
+
+    for announcement in ris_message["announcements"]:
+        for pfx in announcement["prefixes"]:
+            yield BGPElement(
+                type="A",
+                collector=collector,
+                time=timestamp,
+                peer_asn=peer_asn,
+                peer_address=peer_address,
+                fields={
+                    "next-hop": announcement["next_hop"].split(",")[0],
+                    "as-path": path,
+                    "communities": communities,
+                    "prefix": pfx,
+                },
+            )
+
+
+class RISLiveStream:
+    def __init__(
+        self,
+        collectors: list[str],
+        client="pybgpkitstream",
+        filters: FilterOptions = None,
+    ):
+        self.collectors = collectors
+        self.client = client
+        print(filters)
+        self.filters = self._convert_filter_options(filters)
+        print(self.filters)
+
+    @staticmethod
+    def _convert_filter_options(f: FilterOptions) -> dict:
+        """Convert FilterOptions to RIS live filters"""
+        if f is None:
+            return {}
+
+        if not f.model_dump(exclude_unset=True):
+            return {}
+
+        res = {}
+        if f.update_type == "withdraw":
+            res["require"] = "withdrawals"
+        elif f.update_type == "announce":
+            res["require"] = "announcements"
+        if f.peer_ip:
+            res["peer"] = f.peer_ip
+        path_elements = []
+        if f.peer_asn:
+            path_elements.append(f"^{f.peer_asn}")
+        if f.origin_asn:
+            path_elements.append(f"{f.origin_asn}$")
+        res["path"] = ",".join(path_elements)
+
+        if f.prefix:
+            res["prefix"] = f.prefix
+            # default is True which I think is not consistent with BGPKIT/BGPStream
+            res["moreSpecific"] = False
+        if f.prefix_sub:
+            res["prefix"] = f.prefix_sub
+            res["moreSpecific"] = True
+        if f.prefix_super:
+            res["prefix"] = f.prefix_super
+            res["lessSpecific"] = True
+        if f.prefix_super_sub:
+            res["prefix"] = f.prefix_super_sub
+            res["moreSpecific"] = True
+            res["lessSpecific"] = True
+
+        return res
+
+    def __iter__(self) -> Iterator[BGPElement]:
+        ws = websocket.WebSocket()
+        ws.connect(f"wss://ris-live.ripe.net/v1/ws/?client={self.client}")
+
+        # Subscribe to each collector on the same connection
+        for collector in self.collectors:
+            params = {"host": collector, "type": "UPDATE"}
+            params = params | self.filters
+            print(params)
+            ws.send(json.dumps({"type": "ris_subscribe", "data": params}))
+
+        for data in ws:
+            parsed = json.loads(data)["data"]
+            yield from ris_message2bgpelem(parsed)
+
+
+def jitter_buffer_stream(stream, buffer_delay=10) -> Iterator[BGPElement]:
+    """
+    Produces an ordered stream by buffering elements for `buffer_delay` seconds.
+    """
+    heap = []
+    max_ts_seen = float("-inf")
+
+    for elem in stream:
+        # Track the latest timestamp seen in the jittery stream
+        if elem.time > max_ts_seen:
+            max_ts_seen = elem.time
+
+        heapq.heappush(heap, elem)
+
+        # Flush from buffer if timestamp is old enough
+        while heap and (max_ts_seen - heap[0].time) > buffer_delay:
+            yield heapq.heappop(heap)
+
+    # Clean up when stream ends (never hopefully)
+    while heap:
+        yield heapq.heappop(heap)

{pybgpkitstream-0.3.0 → pybgpkitstream-0.4.0}/src/pybgpkitstream/utils.py

@@ -1,6 +1,7 @@
 import datetime
 import re
 
+
 def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datetime:
     match = re.search(pattern, filepath)
     if not match:
@@ -8,4 +9,4 @@ def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datet
     timestamp_str = match.group(1)
     dt = datetime.datetime.strptime(timestamp_str, "%Y%m%d.%H%M")
     dt = dt.replace(tzinfo=datetime.timezone.utc)
-    return dt
+    return dt

pybgpkitstream-0.3.0/src/pybgpkitstream/__init__.py

@@ -1,4 +0,0 @@
-from .bgpstreamconfig import BGPStreamConfig, FilterOptions, PyBGPKITStreamConfig
-from .bgpkitstream import BGPKITStream
-
-__all__ = ["BGPStreamConfig", "FilterOptions", "BGPKITStream", "PyBGPKITStreamConfig"]