pybgpkitstream 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

pybgpkitstream/__init__.py

@@ -1,4 +1,17 @@
-from .bgpstreamconfig import BGPStreamConfig, FilterOptions, PyBGPKITStreamConfig
+from .bgpstreamconfig import (
+    BGPStreamConfig,
+    FilterOptions,
+    PyBGPKITStreamConfig,
+    LiveStreamConfig,
+)
 from .bgpkitstream import BGPKITStream
+from .bgpelement import BGPElement
 
-__all__ = ["BGPStreamConfig", "FilterOptions", "BGPKITStream", "PyBGPKITStreamConfig"]
+__all__ = [
+    "BGPStreamConfig",
+    "FilterOptions",
+    "BGPKITStream",
+    "PyBGPKITStreamConfig",
+    "BGPElement",
+    "LiveStreamConfig",
+]

pybgpkitstream/bgpelement.py

@@ -14,9 +14,9 @@ class ElementFields(TypedDict):
 class BGPElement(NamedTuple):
     """Compatible with pybgpstream.BGPElem"""
 
+    time: float  # time first for sorting tuples convention
     type: str
     collector: str
-    time: float
     peer_asn: int
     peer_address: str
     fields: ElementFields
@@ -31,7 +31,9 @@ class BGPElement(NamedTuple):
             self.peer_address,
             self._maybe_field("prefix"),
             self._maybe_field("next-hop"),
-            self._maybe_field("as-path"),
+            " ".join(map(str, self.fields["as-path"]))
+            if "as-path" in self.fields
+            else None,
             " ".join(self.fields["communities"])
             if "communities" in self.fields
             else None,
@@ -42,3 +44,10 @@ class BGPElement(NamedTuple):
     def _maybe_field(self, field):
         """Credit to pybgpstream"""
         return self.fields[field] if field in self.fields else None
+
+    # Useful for sorting streams
+    def __lt__(self, other):
+        return self.time < other.time
+
+    def __le__(self, other):
+        return self.time <= other.time

pybgpkitstream/bgpkitstream.py

@@ -19,6 +19,7 @@ from pybgpkitstream.bgpstreamconfig import (
     BGPStreamConfig,
     FilterOptions,
     PyBGPKITStreamConfig,
+    LiveStreamConfig,
 )
 from pybgpkitstream.bgpelement import BGPElement
 from pybgpkitstream.bgpparser import (
@@ -28,6 +29,7 @@ from pybgpkitstream.bgpparser import (
     PyBGPStreamParser,
     BGPdumpParser,
 )
+from pybgpkitstream.rislive import RISLiveStream, jitter_buffer_stream
 from pybgpkitstream.utils import dt_from_filepath
 
 name2parser = {
@@ -41,23 +43,6 @@ name2parser = {
 logger = logging.getLogger(__name__)
 
 
-def convert_bgpkit_elem(element, is_rib: bool, collector: str) -> BGPElement:
-    """Convert pybgpkit element to pybgpstream-like element"""
-    return BGPElement(
-        type="R" if is_rib else element.elem_type,
-        collector=collector,
-        time=element.timestamp,
-        peer_asn=element.peer_asn,
-        peer_address=element.peer_ip,
-        fields={
-            "next-hop": element.next_hop,
-            "as-path": element.as_path,
-            "communities": [] if not element.communities else element.communities,
-            "prefix": element.prefix,
-        },
-    )
-
-
 def crc32(input_str: str):
     input_bytes = input_str.encode("utf-8")
     crc = binascii.crc32(input_bytes) & 0xFFFFFFFF
@@ -87,21 +72,22 @@ def get_shared_memory():
 class BGPKITStream:
     def __init__(
         self,
-        ts_start: float,
-        ts_end: float,
-        collector_id: str,
+        collectors: list[str],
         data_type: list[Literal["update", "rib"]],
-        filters: FilterOptions | None,
+        ts_start: float = None,
+        ts_end: float = None,
+        filters: FilterOptions | None = None,
         cache_dir: str | None = None,
         max_concurrent_downloads: int | None = 10,
         chunk_time: float | None = datetime.timedelta(hours=2).seconds,
         ram_fetch: bool | None = True,
         parser_name: str | None = "pybgpkit",
+        jitter_buffer_delay: float | None = 10.0,
     ):
         # Stream config
         self.ts_start = ts_start
         self.ts_end = ts_end
-        self.collector_id = collector_id
+        self.collectors = collectors
         self.data_type = data_type
         if not filters:
             filters = FilterOptions()
@@ -126,6 +112,9 @@ class BGPKITStream:
         self.broker = bgpkit.Broker()
         self.parser_cls: BGPParser = name2parser[parser_name]
 
+        # Live config
+        self.jitter_buffer_delay = jitter_buffer_delay
+
     @staticmethod
     def _generate_cache_filename(url):
         """Generate a cache filename compatible with BGPKIT parser."""
@@ -163,7 +152,7 @@ class BGPKITStream:
             items: list[BrokerItem] = self.broker.query(
                 ts_start=int(self.ts_start - 60),
                 ts_end=int(self.ts_end),
-                collector_id=self.collector_id,
+                collector_id=",".join(self.collectors),
                 data_type=data_type,
             )
             for item in items:
@@ -226,6 +215,8 @@ class BGPKITStream:
                 logging.info("All downloads finished.")
 
     def __iter__(self):
+        if self.ts_start is None and self.ts_end is None:
+            return self._iter_live()
         if "update" in self.data_type:
             return self._iter_update()
         else:
@@ -250,7 +241,7 @@ class BGPKITStream:
                         ts_start=current,
                         ts_end=chunk_end
                         - 1,  # remove one second because BGPKIT include border
-                        collector_id=self.collector_id,
+                        collectors=self.collectors,
                         data_type=self.data_type,
                         cache_dir=self.cache_dir.name
                         if isinstance(self.cache_dir, Directory)
@@ -319,7 +310,7 @@ class BGPKITStream:
                         ts_start=current,
                         ts_end=chunk_end
                         - 1,  # remove one second because BGPKIT include border
-                        collector_id=self.collector_id,
+                        collectors=self.collectors,
                         data_type=self.data_type,
                         cache_dir=self.cache_dir.name
                         if isinstance(self.cache_dir, Directory)
@@ -365,14 +356,30 @@ class BGPKITStream:
         finally:
             self.cache_dir.cleanup()
 
+    def _iter_live(self) -> Iterator[BGPElement]:
+
+        ris_collectors = [
+            collector for collector in self.collectors if collector[:3] == "rrc"
+        ]
+
+        stream = RISLiveStream(collectors=ris_collectors, filters=self.filters)
+
+        if self.jitter_buffer_delay is not None and self.jitter_buffer_delay > 0:
+            stream = jitter_buffer_stream(stream, buffer_delay=self.jitter_buffer_delay)
+
+        for elem in stream:
+            yield elem
+
     @classmethod
-    def from_config(cls, config: PyBGPKITStreamConfig | BGPStreamConfig):
+    def from_config(
+        cls, config: PyBGPKITStreamConfig | BGPStreamConfig | LiveStreamConfig
+    ):
         if isinstance(config, PyBGPKITStreamConfig):
             stream_config = config.bgpstream_config
             return cls(
                 ts_start=stream_config.start_time.timestamp(),
                 ts_end=stream_config.end_time.timestamp(),
-                collector_id=",".join(stream_config.collectors),
+                collectors=stream_config.collectors,
                 data_type=[dtype[:-1] for dtype in stream_config.data_types],
                 filters=stream_config.filters
                 if stream_config.filters
@@ -387,10 +394,29 @@ class BGPKITStream:
             )
 
         elif isinstance(config, BGPStreamConfig):
+            if not config.is_live():
+                return cls(
+                    ts_start=config.start_time.timestamp(),
+                    ts_end=config.end_time.timestamp(),
+                    collectors=config.collectors,
+                    data_type=[dtype[:-1] for dtype in config.data_types],
+                    filters=config.filters if config.filters else FilterOptions(),
+                )
+            else:
+                return cls(
+                    collectors=config.collectors,
+                    data_type=["update"],
+                    filters=config.filters if config.filters else FilterOptions(),
+                    jitter_buffer_delay=10,
+                )
+
+        elif isinstance(config, LiveStreamConfig):
             return cls(
-                ts_start=config.start_time.timestamp(),
-                ts_end=config.end_time.timestamp(),
-                collector_id=",".join(config.collectors),
-                data_type=[dtype[:-1] for dtype in config.data_types],
+                collectors=config.collectors,
+                data_type=["update"],
                 filters=config.filters if config.filters else FilterOptions(),
+                jitter_buffer_delay=config.jitter_buffer_delay,
             )
+
+        else:
+            raise ValueError("Unsupported config type")

pybgpkitstream/bgpparser.py

@@ -6,6 +6,7 @@ import re
 import ipaddress
 import subprocess as sp
 from pybgpkitstream.utils import dt_from_filepath
+import logging
 
 try:
     import pybgpstream
@@ -36,12 +37,21 @@ class PyBGPKITParser(BGPParser):
         self.parser = None  # placeholder for lazy instantiation
         self.is_rib = is_rib
         self.collector = collector
-        self.filters = filters.model_dump(exclude_unset=True)
+        self.filters: dict = filters.model_dump(exclude_unset=True, exclude_none=True)
         # cast int ipv to pybgpkit ipv4 or ipv6 string
         if "ip_version" in self.filters:
             ipv_int = self.filters["ip_version"]
             if ipv_int:
                 self.filters["ip_version"] = f"ipv{ipv_int}"
+        if self.filters.get("peer_asn"):
+            self.filters["peer_asn"] = str(self.filters["peer_asn"])
+        if self.filters.get("origin_asn"):
+            self.filters["origin_asn"] = str(self.filters["origin_asn"])
+        if self.filters.get("update_type"):
+            val = self.filters.pop("update_type")
+            self.filters["type"] = val
+        if self.filters.get("peer_ips"):
+            self.filters["peer_ips"] = ", ".join(self.filters["peer_ips"])
 
     def _convert(self, element) -> BGPElement:
         return BGPElement(
@@ -105,9 +115,9 @@ class BGPKITParser(BGPParser):
         # Structure: Type|Time|PeerIP|PeerAS|Prefix
         if rec_type == "W":
             return BGPElement(
+                time=self.time,  # force RIB filename timestamp instead of last changed
                 type="W",
                 collector=self.collector,
-                time=self.time,  # force RIB filename timestamp instead of last changed
                 peer_asn=int(element[3]),
                 peer_address=element[2],
                 fields={"prefix": element[4]},
@@ -123,11 +133,10 @@ class BGPKITParser(BGPParser):
 
         return BGPElement(
             # bgpkit outputs 'A' for both Updates and RIB entries.
-            # Map to "A" (Announcement) or change to "R" if you strictly need RIB typing.
+            self.time,
             "R" if self.is_rib else rec_type,
             self.collector,
             # float(element[1]),
-            self.time,
             int(element[3]),
             element[2],
             {
@@ -154,16 +163,44 @@ class PyBGPStreamParser(BGPParser):
     ):
         self.filepath = filepath
         self.collector = collector
-        self.filters = generate_bgpstream_filters(filters) if filters else None
+        self.filters = filters
 
-    def __iter__(self):
-        stream = pybgpstream.BGPStream(data_interface="singlefile", filter=self.filters)
+    def _iter_normal(self):
+        """when there is no filter or filters are supported by pybgpstream"""
+        stream = pybgpstream.BGPStream(
+            data_interface="singlefile",
+            filter=generate_bgpstream_filters(self.filters) if self.filters else None,
+        )
         stream.set_data_interface_option("singlefile", "rib-file", self.filepath)
 
         for elem in stream:
             elem.collector = self.collector
             yield elem
 
+    def _iter_python_filter(self):
+        """when filters are not supported by pybgpstream, filter from the python side"""
+        bgpstream_filter = generate_bgpstream_filters(self.filters)
+        stream = pybgpstream.BGPStream(
+            data_interface="singlefile",
+            filter=bgpstream_filter if bgpstream_filter else None,
+        )
+        stream.set_data_interface_option("singlefile", "rib-file", self.filepath)
+        peer_ips = set(self.filters.peer_ips)
+
+        for elem in stream:
+            if elem.peer_address not in peer_ips:
+                continue
+            elem.collector = self.collector
+            yield elem
+
+    def __iter__(self):
+        if not self.filters.peer_ip and not self.filters.peer_ips:
+            return self._iter_normal()
+        else:
+            if self.filters.peer_ip:
+                self.filters.peer_ips = [self.filters.peer_ip]
+            return self._iter_python_filter()
+
 
 class BGPdumpParser(BGPParser):
     """Run bgpdump as a subprocess. I might have over-engineered the filtering."""
@@ -204,9 +241,9 @@ class BGPdumpParser(BGPParser):
         # 1. Handle Withdrawals (Fastest path, fewer fields)
         if elem_type == "W":
             return BGPElement(
+                float(element[1]),
                 "W",
                 self.collector,
-                float(element[1]),
                 int(element[4]),
                 element[3],
                 {"prefix": element[5]},  # Dict literal is faster than assignment
@@ -219,9 +256,9 @@ class BGPdumpParser(BGPParser):
         # Logic: if TABLE_DUMP2, type is R, else A
         # Construct fields dict in one shot (BUILD_MAP opcode)
         return BGPElement(
+            float(element[1]),
             "R" if elem_type == "B" else "A",
             self.collector,
-            float(element[1]),
             int(element[4]),
             element[3],
             {
@@ -229,7 +266,7 @@ class BGPdumpParser(BGPParser):
                 "as-path": element[6],
                 "next-hop": element[8],
                 # Check for empty string before splitting (avoids creating [''])
-                "communities": rec_comm.split() if rec_comm else [],
+                "communities": rec_comm.split(" ") if rec_comm else [],
             },
         )
 
@@ -362,12 +399,12 @@ def generate_bgpstream_filters(f: FilterOptions) -> str | None:
         parts.append(f"prefix any {f.prefix_super_sub}")
 
     if f.ip_version:
-        parts.append(f"ipversion {f.ip_version[-1]}")
+        parts.append(f"ipversion {f.ip_version}")
 
     # Warn about unsupported fields
     if f.peer_ip or f.peer_ips:
-        print(
-            "Warning: peer_ip and peer_ips are not supported by this BGPStream filter string parser and will be ignored."
+        logging.info(
+            "Filtering by peer_ip is not supported natively by pybgpstream (falling back to python-side filtering)"
         )
 
     # Join all parts with 'and' as required by the parser

pybgpkitstream/bgpstreamconfig.py

@@ -48,20 +48,27 @@ class FilterOptions(BaseModel):
 
 
 class BGPStreamConfig(BaseModel):
-    """Unified BGPStream config, compatible with BGPKIT and pybgpstream"""
+    """Unified BGPStream config"""
 
-    start_time: datetime.datetime = Field(description="Start of the stream")
-    end_time: datetime.datetime = Field(description="End of the stream")
+    start_time: datetime.datetime | None = Field(
+        default=None, description="Start of the stream"
+    )
+    end_time: datetime.datetime | None = Field(
+        default=None, description="End of the stream"
+    )
     collectors: list[str] = Field(description="List of collectors to get data from")
-    data_types: list[Literal["ribs", "updates"]] = Field(
-        description="List of archives files to consider (`ribs` or `updates`)"
+    data_types: list[Literal["ribs", "updates"]] | None = Field(
+        default=["updates"],
+        description="List of archives files to consider (`ribs` or `updates`)",
     )
 
     filters: FilterOptions | None = Field(default=None, description="Optional filters")
 
-    @field_validator("start_time", "end_time")
+    @field_validator("start_time", "end_time", mode="before")
     @classmethod
     def normalize_to_utc(cls, dt: datetime.datetime) -> datetime.datetime:
+        if dt is None:
+            return None
         # if naive datetime (not timezone-aware) assume it's UTC
         if dt.tzinfo is None:
             return dt.replace(tzinfo=datetime.timezone.utc)
@@ -69,6 +76,38 @@ class BGPStreamConfig(BaseModel):
         else:
             return dt.astimezone(datetime.timezone.utc)
 
+    @model_validator(mode="after")
+    def validate(self) -> "BGPStreamConfig":
+
+        if (self.start_time is None) ^ (self.end_time is None):
+            raise ValueError(
+                "Provide both start and end times, or leave both as None for live mode."
+            )
+        if not self.is_live():
+            assert self.start_time < self.end_time
+        # Force data_type to update for live mode
+        else:
+            if self.data_types is None:
+                self.data_types = ["updates"]
+
+        return self
+
+    def is_live(self) -> bool:
+        return self.start_time is None and self.end_time is None
+
+
+class LiveStreamConfig(BaseModel):
+    """Config for live mode"""
+
+    collectors: list[str] = Field(
+        description="List of collectors to get data from (for now only RIS live collectors)"
+    )
+    filters: FilterOptions | None = Field(default=None, description="Optional filters")
+    jitter_buffer_delay: float | None = Field(
+        default=10.0,
+        description="Jitter buffer time in seconds to make sure RIS live updates are time-sorted. Introduce a slight delay. Set to None or 0 to disable",
+    )
+
 
 class PyBGPKITStreamConfig(BaseModel):
     """Unified BGPStream config and parameters related to PyBGPKIT implementation (all optional)"""
@@ -135,26 +174,31 @@ class PyBGPKITStreamConfig(BaseModel):
                 raise ValueError(
                     "bgpkit binary not found in PATH. "
                     "Install from: https://github.com/bgpkit/bgpkit-parser "
-                    "or use cargo: cargo install bgpkit-parser"
+                    "or use cargo: cargo install bgpkit-parser --features cli"
                 )
 
-        # Return the parser value if validation passes
         return parser
-    
-    @model_validator(mode='before')
+
+    @model_validator(mode="before")
     @classmethod
     def nest_bgpstream_params(cls, data: dict) -> dict:
         """Allow to define a flat config"""
         # If the user already provided 'bgpstream_config', do nothing
         if "bgpstream_config" in data:
             return data
-        
+
         # Define which fields belong to the inner BGPStreamConfig
-        stream_fields = {"start_time", "end_time", "collectors", "data_types", "filters"}
-        
+        stream_fields = {
+            "start_time",
+            "end_time",
+            "collectors",
+            "data_types",
+            "filters",
+        }
+
         # Extract those fields from the flat input
         inner_data = {k: data.pop(k) for k in stream_fields if k in data}
-        
+
         # Nest them back into the dictionary
         data["bgpstream_config"] = inner_data
         return data

pybgpkitstream/cli.py

@@ -2,8 +2,12 @@ import argparse
 import sys
 import datetime
 
-from pybgpkitstream import BGPStreamConfig, FilterOptions
-from pybgpkitstream import BGPKITStream
+from pybgpkitstream import (
+    BGPStreamConfig,
+    FilterOptions,
+    PyBGPKITStreamConfig,
+    BGPKITStream,
+)
 
 
 def main():
@@ -40,12 +44,6 @@ def main():
         default=["updates"],
         help="List of archives to consider ('ribs' or 'updates').",
     )
-    parser.add_argument(
-        "--cache-dir",
-        type=str,
-        default=None,
-        help="Directory for caching downloaded files.",
-    )
 
     # Arguments for FilterOptions
     parser.add_argument(
@@ -93,7 +91,7 @@ def main():
     )
     parser.add_argument(
         "--peer-asn",
-        type=str,
+        type=int,
         default=None,
         help="Filter by the AS number of the BGP peer.",
     )
@@ -111,6 +109,20 @@ def main():
         help="Filter by a regular expression matching the AS path.",
     )
 
+    # PyBGPKITStream implementation parameters
+    parser.add_argument(
+        "--cache-dir",
+        type=str,
+        default=None,
+        help="Directory for caching downloaded files.",
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        choices=["pybgpkit", "bgpkit", "pybgpstream", "bgpdump"],
+        default="pybgpkit",
+    )
+
     args = parser.parse_args()
 
     filter_options = FilterOptions(
@@ -130,15 +142,18 @@ def main():
     if all(value is None for value in filter_options.model_dump().values()):
         filter_options = None
 
-    config = BGPStreamConfig(
+    bgpstream_config = BGPStreamConfig(
         start_time=args.start_time,
         end_time=args.end_time,
         collectors=args.collectors,
         data_types=args.data_types,
-        cache_dir=args.cache_dir,
         filters=filter_options,
     )
 
+    config = PyBGPKITStreamConfig(
+        bgpstream_config=bgpstream_config, cache_dir=args.cache_dir, parser=args.parser
+    )
+
     try:
         for element in BGPKITStream.from_config(config):
             print(element)

pybgpkitstream/rislive.py

@@ -0,0 +1,141 @@
+from typing import Iterator
+import json
+import heapq
+import websocket
+
+from pybgpkitstream.bgpelement import BGPElement
+from pybgpkitstream.bgpstreamconfig import FilterOptions
+
+
+def ris_message2bgpelem(ris_message: dict) -> Iterator[BGPElement]:
+
+    timestamp = float(ris_message["timestamp"])
+    collector = ris_message["host"].split(".")[0]
+    peer_asn = int(ris_message["peer_asn"])
+    peer_address = ris_message["peer"]
+    path = ris_message["path"]
+    communities = ris_message["community"]
+    if communities:
+        communities = [f"{asn}:{community}" for asn, community in communities]
+
+    for pfx in ris_message["withdrawals"]:
+        yield BGPElement(
+            type="W",
+            collector=collector,
+            time=timestamp,
+            peer_asn=peer_asn,
+            peer_address=peer_address,
+            fields={
+                "as-path": path,
+                "communities": communities,
+                "prefix": pfx,
+            },
+        )
+
+    for announcement in ris_message["announcements"]:
+        for pfx in announcement["prefixes"]:
+            yield BGPElement(
+                type="A",
+                collector=collector,
+                time=timestamp,
+                peer_asn=peer_asn,
+                peer_address=peer_address,
+                fields={
+                    "next-hop": announcement["next_hop"].split(",")[0],
+                    "as-path": path,
+                    "communities": communities,
+                    "prefix": pfx,
+                },
+            )
+
+
+class RISLiveStream:
+    def __init__(
+        self,
+        collectors: list[str],
+        client="pybgpkitstream",
+        filters: FilterOptions = None,
+    ):
+        self.collectors = collectors
+        self.client = client
+        print(filters)
+        self.filters = self._convert_filter_options(filters)
+        print(self.filters)
+
+    @staticmethod
+    def _convert_filter_options(f: FilterOptions) -> dict:
+        """Convert FilterOptions to RIS live filters"""
+        if f is None:
+            return {}
+
+        if not f.model_dump(exclude_unset=True):
+            return {}
+
+        res = {}
+        if f.update_type == "withdraw":
+            res["require"] = "withdrawals"
+        elif f.update_type == "announce":
+            res["require"] = "announcements"
+        if f.peer_ip:
+            res["peer"] = f.peer_ip
+        path_elements = []
+        if f.peer_asn:
+            path_elements.append(f"^{f.peer_asn}")
+        if f.origin_asn:
+            path_elements.append(f"{f.origin_asn}$")
+        res["path"] = ",".join(path_elements)
+
+        if f.prefix:
+            res["prefix"] = f.prefix
+            # default is True which I think is not consistent with BGPKIT/BGPStream
+            res["moreSpecific"] = False
+        if f.prefix_sub:
+            res["prefix"] = f.prefix_sub
+            res["moreSpecific"] = True
+        if f.prefix_super:
+            res["prefix"] = f.prefix_super
+            res["lessSpecific"] = True
+        if f.prefix_super_sub:
+            res["prefix"] = f.prefix_super_sub
+            res["moreSpecific"] = True
+            res["lessSpecific"] = True
+
+        return res
+
+    def __iter__(self) -> Iterator[BGPElement]:
+        ws = websocket.WebSocket()
+        ws.connect(f"wss://ris-live.ripe.net/v1/ws/?client={self.client}")
+
+        # Subscribe to each collector on the same connection
+        for collector in self.collectors:
+            params = {"host": collector, "type": "UPDATE"}
+            params = params | self.filters
+            print(params)
+            ws.send(json.dumps({"type": "ris_subscribe", "data": params}))
+
+        for data in ws:
+            parsed = json.loads(data)["data"]
+            yield from ris_message2bgpelem(parsed)
+
+
+def jitter_buffer_stream(stream, buffer_delay=10) -> Iterator[BGPElement]:
+    """
+    Produces an ordered stream by buffering elements for `buffer_delay` seconds.
+    """
+    heap = []
+    max_ts_seen = float("-inf")
+
+    for elem in stream:
+        # Track the latest timestamp seen in the jittery stream
+        if elem.time > max_ts_seen:
+            max_ts_seen = elem.time
+
+        heapq.heappush(heap, elem)
+
+        # Flush from buffer if timestamp is old enough
+        while heap and (max_ts_seen - heap[0].time) > buffer_delay:
+            yield heapq.heappop(heap)
+
+    # Clean up when stream ends (never hopefully)
+    while heap:
+        yield heapq.heappop(heap)

pybgpkitstream/utils.py

@@ -1,6 +1,7 @@
 import datetime
 import re
 
+
 def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datetime:
     match = re.search(pattern, filepath)
     if not match:
@@ -8,4 +9,4 @@ def dt_from_filepath(filepath: str, pattern=r"(\d{8}\.\d{4})") -> datetime.datet
     timestamp_str = match.group(1)
     dt = datetime.datetime.strptime(timestamp_str, "%Y%m%d.%H%M")
     dt = dt.replace(tzinfo=datetime.timezone.utc)
-    return dt
+    return dt

pybgpkitstream-0.4.0.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.3
+Name: pybgpkitstream
+Version: 0.4.0
+Summary: Drop-in replacement for PyBGPStream using BGPKIT
+Author: JustinLoye
+Author-email: JustinLoye <jloye@iij.ad.jp>
+Requires-Dist: aiohttp>=3.12.15
+Requires-Dist: pybgpkit>=0.6.2
+Requires-Dist: pydantic>=2.11.9
+Requires-Dist: websocket-client>=1.8.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# PyBGPKITStream
+
+A drop-in replacement for PyBGPStream using BGPKIT
+
+## Features
+
+- Generates time-ordered BGP messages on the fly from RIBs and updates MRT files of multiple collectors
+- Stream the same BGP messages as PyBGPStream, enabling seamless, drop-in replacement
+- Lazy loading consumes minimal memory, making it suitable for large datasets
+- Multiple BGP parsers supported: `pybgpkit` (default but slow), `bgpkit-parser`, `bgpdump` and `pybgpstream` single file backend (the latter three are system dependencies)
+- Caching with concurrent downloading fully compatible with the BGPKIT parser's caching functionality.
+- Performance: for updates, typically 3–10× faster than PyBGPStream; for RIB-only processing, currently about 3–4× slower (see [perf.md](perf.md) for test details).
+- A CLI tool
+
+## Quick start
+
+Installation:
+
+```sh
+pip install pybgpkitstream
+```
+
+Usage:
+
+```python
+import datetime
+from pybgpkitstream import BGPStreamConfig, BGPKITStream
+
+config = BGPStreamConfig(
+    start_time=datetime.datetime(2010, 9, 1, 0, 0),
+    end_time=datetime.datetime(2010, 9, 1, 1, 59),
+    collectors=["route-views.sydney", "route-views.wide"],
+    data_types=["ribs", "updates"],
+)
+
+stream = BGPKITStream.from_config(config)
+
+n_elems = 0
+for elem in stream:
+  n_elems += 1
+    
+print(f"Processed {n_elems} BGP elements")
+```
+
+or in the terminal:
+
+```sh
+pybgpkitstream --start-time 2010-09-01T00:00:00 --end-time 2010-09-01T01:59:00 --collectors route-views.sydney route-views.wide --data-types updates > updates.txt
+```
+
+## Motivation
+
+PyBGPStream is great but the implementation is complex and stops working when UC San Diego experiences a power outage.  
+BGPKIT broker and parser are great, but cannot be used to create an ordered stream of BGP messages from multiple collectors and multiple data types.
+
+## Missing features
+
+- Live mode for RouteViews collectors
+- Some PyBGPStream data interface options like csv or sqlite

pybgpkitstream-0.4.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+pybgpkitstream/__init__.py,sha256=_i1D2bx1SanDe8BZ8B4S1349an7-1JW4ptsY5T8gig4,344
+pybgpkitstream/bgpelement.py,sha256=Kr4YLk14vRZtQeYQv_k-NcODmgP0MHdXaIlUzeRERjk,1466
+pybgpkitstream/bgpkitstream.py,sha256=Bw7V4RYg7Bmkkl2XG4LPiFZGEECOkijcd59EUxsBBbQ,16031
+pybgpkitstream/bgpparser.py,sha256=uQJmbaRQOXsCkTnTW8Lhi336cLrcMWCaoAyNe4ZE1B0,16177
+pybgpkitstream/bgpstreamconfig.py,sha256=PaXMREUFb6zj9Y4Zpnlb1nekV8zGYIyEtvSp99hDuic,7480
+pybgpkitstream/cli.py,sha256=U1jFjEwkuySk7OhUR2sYWkiYFEx2XDLGsSwZM70zGcE,4548
+pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pybgpkitstream/rislive.py,sha256=e0o8qF6nT_hJnDbU7f8ZuPdTwEnO0H3lHlJMqp3rqlU,4437
+pybgpkitstream/utils.py,sha256=BzPUKRdJ48bP73eT4LWByrmN0IIIUdBYuATINMXWpTE,406
+pybgpkitstream-0.4.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+pybgpkitstream-0.4.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
+pybgpkitstream-0.4.0.dist-info/METADATA,sha256=F4sExUM0TL01r8MmV6bftKQWrkH93i18TyreJDwuIuM,2320
+pybgpkitstream-0.4.0.dist-info/RECORD,,

pybgpkitstream-0.2.0.dist-info/METADATA

@@ -1,79 +0,0 @@
-Metadata-Version: 2.3
-Name: pybgpkitstream
-Version: 0.2.0
-Summary: Drop-in replacement for PyBGPStream using BGPKIT
-Author: JustinLoye
-Author-email: JustinLoye <jloye@iij.ad.jp>
-Requires-Dist: aiohttp>=3.12.15
-Requires-Dist: pybgpkit>=0.6.2
-Requires-Dist: pydantic>=2.11.9
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-
-# PyBGPKITStream
-
-A drop-in replacement for PyBGPStream using BGPKIT
-
-## Features
-
-- Effortless BGP Stream Switching:
-  - Seamless, drop-in replacement ([example](tests/test_stream.py#L38))
-  - Lazy message generation: generates time-ordered BGP messages on the fly, consuming minimal memory and making it suitable for large datasets
-  - Supports multiple route collectors
-  - Supports both ribs and updates
-- Caching with concurrent downloading is enabled and is fully compatible with the BGPKIT parser's caching functionality.
-- [Similar performance to PyBGPStream](examples/perf.ipynb)
-- A CLI tool
-
-## Quick start
-
-Installation:
-
-```sh
-pip install pybgpkitstream
-```
-
-Usage:
-
-```python
-import datetime
-from pybgpkitstream import BGPStreamConfig, BGPKITStream
-
-config = BGPStreamConfig(
-    start_time=datetime.datetime(2010, 9, 1, 0, 0),
-    end_time=datetime.datetime(2010, 9, 1, 1, 59),
-    collectors=["route-views.sydney", "route-views.wide"],
-    data_types=["ribs", "updates"],
-)
-
-stream = BGPKITStream.from_config(config)
-
-n_elems = 0
-for _ in stream:
-    n_elems += 1
-    
-print(f"Processed {n_elems} BGP elements")
-```
-
-or in the terminal:
-
-```sh
-pybgpkitstream --start-time 2010-09-01T00:00:00 --end-time 2010-09-01T01:59:00 --collectors route-views.sydney route-views.wide --data-types updates > updates.txt
-```
-
-## Motivation
-
-PyBGPStream is great but the implementation is complex and stops working when UC San Diego experiences a power outage.
-BGPKIT broker and parser are great, but cannot be used to create an ordered stream of BGP messages from multiple collectors and multiple data types.
-
-## Missing features
-
-- live mode
-- `pybgpkitstream.BGPElement` is not fully compatible with `pybgpstream.BGPElem`: missing record_type (BGPKIT limitation), project (BGPKIT limitation), router (could be improved), router_ip (could be improved)
-- CLI output is not yet compatible with `bgpdump -m` or `bgpreader` (right now a similar-looking output is produced)
-
-## Issues
-
-- Program will crash when working with many update files per collector (~ more than few hours of data), only when caching is disabled. This might be caused by [BGPKIT parser not being lazy](https://github.com/bgpkit/bgpkit-parser/pull/239). See [details and workaround fix](examples/many_updates.ipynb)
-- Filters are designed with BGPKIT in mind, and can slightly differ to pybgpstream. See [this file](tests/pybgpstream_utils.py) for a conversion to PyBGPStream filter. Note that for now the filters have not been heavily tested...
-- ... just like the rest of the project. Use at your own risk. The only tests I did are in /tests

pybgpkitstream-0.2.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-pybgpkitstream/__init__.py,sha256=OGWVhZdSvialNkIkQ1VBrmiyOcwkCA1D5IaLo7WQnPI,209
-pybgpkitstream/bgpelement.py,sha256=7mXSUmWThhIbKy2JVsLchoteve0BshT3uH8cdbAe0Go,1176
-pybgpkitstream/bgpkitstream.py,sha256=CKQv7dU-ooznuD1AjHKnZ6qRdPH1ZiOIEGtVNtU8PCY,15062
-pybgpkitstream/bgpparser.py,sha256=aJcVCv_ydy3xQcH_BBxQE4hc7G1rLYqqNJAXCdnrasA,14689
-pybgpkitstream/bgpstreamconfig.py,sha256=vIfEN475WDIZ7kGmi3dnj_1GIQE_r6qkDFETZmMvH5E,6199
-pybgpkitstream/cli.py,sha256=E0E1hO0fzGhy1skBopRufdewsiSy_mA-J8Gf2WxBRxo,4214
-pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pybgpkitstream/utils.py,sha256=6FwEEpBtY_20BDlJPOPFmTYQGqw7fCBLjXmnd7gjBdQ,404
-pybgpkitstream-0.2.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-pybgpkitstream-0.2.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
-pybgpkitstream-0.2.0.dist-info/METADATA,sha256=EBkydfiRKhgjAz4NotQqix2wX9uSHLK3iDQ_1oUSEqU,2953
-pybgpkitstream-0.2.0.dist-info/RECORD,,