pybgpkitstream 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

pybgpkitstream/bgpparser.py

@@ -6,6 +6,7 @@ import re
 import ipaddress
 import subprocess as sp
 from pybgpkitstream.utils import dt_from_filepath
+import logging
 
 try:
     import pybgpstream
@@ -36,12 +37,21 @@ class PyBGPKITParser(BGPParser):
         self.parser = None  # placeholder for lazy instantiation
         self.is_rib = is_rib
         self.collector = collector
-        self.filters = filters.model_dump(exclude_unset=True)
+        self.filters: dict = filters.model_dump(exclude_unset=True, exclude_none=True)
         # cast int ipv to pybgpkit ipv4 or ipv6 string
         if "ip_version" in self.filters:
             ipv_int = self.filters["ip_version"]
             if ipv_int:
                 self.filters["ip_version"] = f"ipv{ipv_int}"
+        if self.filters.get("peer_asn"):
+            self.filters["peer_asn"] = str(self.filters["peer_asn"])
+        if self.filters.get("origin_asn"):
+            self.filters["origin_asn"] = str(self.filters["origin_asn"])
+        if self.filters.get("update_type"):
+            val = self.filters.pop("update_type")
+            self.filters["type"] = val
+        if self.filters.get("peer_ips"):
+            self.filters["peer_ips"] = ", ".join(self.filters["peer_ips"])
 
     def _convert(self, element) -> BGPElement:
         return BGPElement(
@@ -154,16 +164,44 @@ class PyBGPStreamParser(BGPParser):
     ):
         self.filepath = filepath
         self.collector = collector
-        self.filters = generate_bgpstream_filters(filters) if filters else None
+        self.filters = filters
 
-    def __iter__(self):
-        stream = pybgpstream.BGPStream(data_interface="singlefile", filter=self.filters)
+    def _iter_normal(self):
+        """when there is no filter or filters are supported by pybgpstream"""
+        stream = pybgpstream.BGPStream(
+            data_interface="singlefile",
+            filter=generate_bgpstream_filters(self.filters) if self.filters else None,
+        )
+        stream.set_data_interface_option("singlefile", "rib-file", self.filepath)
+
+        for elem in stream:
+            elem.collector = self.collector
+            yield elem
+
+    def _iter_python_filter(self):
+        """when filters are not supported by pybgpstream, filter from the python side"""
+        bgpstream_filter = generate_bgpstream_filters(self.filters)
+        stream = pybgpstream.BGPStream(
+            data_interface="singlefile",
+            filter=bgpstream_filter if bgpstream_filter else None,
+        )
         stream.set_data_interface_option("singlefile", "rib-file", self.filepath)
+        peer_ips = set(self.filters.peer_ips)
 
         for elem in stream:
+            if elem.peer_address not in peer_ips:
+                continue
             elem.collector = self.collector
             yield elem
 
+    def __iter__(self):
+        if not self.filters.peer_ip and not self.filters.peer_ips:
+            return self._iter_normal()
+        else:
+            if self.filters.peer_ip:
+                self.filters.peer_ips = [self.filters.peer_ip]
+            return self._iter_python_filter()
+
 
 class BGPdumpParser(BGPParser):
     """Run bgpdump as a subprocess. I might have over-engineered the filtering."""
@@ -362,12 +400,12 @@ def generate_bgpstream_filters(f: FilterOptions) -> str | None:
         parts.append(f"prefix any {f.prefix_super_sub}")
 
     if f.ip_version:
-        parts.append(f"ipversion {f.ip_version[-1]}")
+        parts.append(f"ipversion {f.ip_version}")
 
     # Warn about unsupported fields
     if f.peer_ip or f.peer_ips:
-        print(
-            "Warning: peer_ip and peer_ips are not supported by this BGPStream filter string parser and will be ignored."
+        logging.info(
+            "Filtering by peer_ip is not supported natively by pybgpstream (falling back to python-side filtering)"
         )
 
     # Join all parts with 'and' as required by the parser

pybgpkitstream/cli.py

@@ -2,8 +2,12 @@ import argparse
 import sys
 import datetime
 
-from pybgpkitstream import BGPStreamConfig, FilterOptions
-from pybgpkitstream import BGPKITStream
+from pybgpkitstream import (
+    BGPStreamConfig,
+    FilterOptions,
+    PyBGPKITStreamConfig,
+    BGPKITStream,
+)
 
 
 def main():
@@ -40,12 +44,6 @@ def main():
         default=["updates"],
         help="List of archives to consider ('ribs' or 'updates').",
     )
-    parser.add_argument(
-        "--cache-dir",
-        type=str,
-        default=None,
-        help="Directory for caching downloaded files.",
-    )
 
     # Arguments for FilterOptions
     parser.add_argument(
@@ -93,7 +91,7 @@ def main():
     )
     parser.add_argument(
         "--peer-asn",
-        type=str,
+        type=int,
         default=None,
         help="Filter by the AS number of the BGP peer.",
     )
@@ -111,6 +109,20 @@ def main():
         help="Filter by a regular expression matching the AS path.",
     )
 
+    # PyBGPKITStream implementation parameters
+    parser.add_argument(
+        "--cache-dir",
+        type=str,
+        default=None,
+        help="Directory for caching downloaded files.",
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        choices=["pybgpkit", "bgpkit", "pybgpstream", "bgpdump"],
+        default="pybgpkit",
+    )
+
     args = parser.parse_args()
 
     filter_options = FilterOptions(
@@ -130,19 +142,25 @@ def main():
     if all(value is None for value in filter_options.model_dump().values()):
         filter_options = None
 
-    config = BGPStreamConfig(
+    bgpstream_config = BGPStreamConfig(
         start_time=args.start_time,
         end_time=args.end_time,
         collectors=args.collectors,
         data_types=args.data_types,
-        cache_dir=args.cache_dir,
         filters=filter_options,
     )
 
+    config = PyBGPKITStreamConfig(
+        bgpstream_config=bgpstream_config, cache_dir=args.cache_dir, parser=args.parser
+    )
+
+    for element in BGPKITStream.from_config(config):
+        print(element)
     try:
         for element in BGPKITStream.from_config(config):
             print(element)
     except Exception as e:
+        print(e)
         print(f"An error occurred during streaming: {e}", file=sys.stderr)
         sys.exit(1)
 

pybgpkitstream-0.3.0.dist-info/METADATA

@@ -0,0 +1,71 @@
+Metadata-Version: 2.3
+Name: pybgpkitstream
+Version: 0.3.0
+Summary: Drop-in replacement for PyBGPStream using BGPKIT
+Author: JustinLoye
+Author-email: JustinLoye <jloye@iij.ad.jp>
+Requires-Dist: aiohttp>=3.12.15
+Requires-Dist: pybgpkit>=0.6.2
+Requires-Dist: pydantic>=2.11.9
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# PyBGPKITStream
+
+A drop-in replacement for PyBGPStream using BGPKIT
+
+## Features
+
+- Generates time-ordered BGP messages on the fly from RIBs and updates MRT files of multiple collectors
+- Stream the same BGP messages as PyBGPStream, enabling seamless, drop-in replacement
+- Lazy loading consumes minimal memory, making it suitable for large datasets
+- Multiple BGP parsers supported: `pybgpkit` (default but slow), `bgpkit-parser`, `bgpdump` and `pybgpstream` single file backend (the latter three are system dependencies)
+- Caching with concurrent downloading fully compatible with the BGPKIT parser's caching functionality.
+- Performance: for updates, typically 3–10× faster than PyBGPStream; for RIB-only processing, currently about 3–4× slower (see [perf.md](perf.md) for test details).
+- A CLI tool
+
+## Quick start
+
+Installation:
+
+```sh
+pip install pybgpkitstream
+```
+
+Usage:
+
+```python
+import datetime
+from pybgpkitstream import BGPStreamConfig, BGPKITStream
+
+config = BGPStreamConfig(
+    start_time=datetime.datetime(2010, 9, 1, 0, 0),
+    end_time=datetime.datetime(2010, 9, 1, 1, 59),
+    collectors=["route-views.sydney", "route-views.wide"],
+    data_types=["ribs", "updates"],
+)
+
+stream = BGPKITStream.from_config(config)
+
+n_elems = 0
+for elem in stream:
+  n_elems += 1
+    
+print(f"Processed {n_elems} BGP elements")
+```
+
+or in the terminal:
+
+```sh
+pybgpkitstream --start-time 2010-09-01T00:00:00 --end-time 2010-09-01T01:59:00 --collectors route-views.sydney route-views.wide --data-types updates > updates.txt
+```
+
+## Motivation
+
+PyBGPStream is great but the implementation is complex and stops working when UC San Diego experiences a power outage.  
+BGPKIT broker and parser are great, but cannot be used to create an ordered stream of BGP messages from multiple collectors and multiple data types.
+
+## Missing features
+
+- live mode (I plan to add semi-live soon.)
+- `pybgpkitstream.BGPElement` is not fully compatible with `pybgpstream.BGPElem`: missing record_type, project, router, router_ip

{pybgpkitstream-0.2.0.dist-info → pybgpkitstream-0.3.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
 pybgpkitstream/__init__.py,sha256=OGWVhZdSvialNkIkQ1VBrmiyOcwkCA1D5IaLo7WQnPI,209
 pybgpkitstream/bgpelement.py,sha256=7mXSUmWThhIbKy2JVsLchoteve0BshT3uH8cdbAe0Go,1176
 pybgpkitstream/bgpkitstream.py,sha256=CKQv7dU-ooznuD1AjHKnZ6qRdPH1ZiOIEGtVNtU8PCY,15062
-pybgpkitstream/bgpparser.py,sha256=aJcVCv_ydy3xQcH_BBxQE4hc7G1rLYqqNJAXCdnrasA,14689
+pybgpkitstream/bgpparser.py,sha256=eDTWV6iGZTxgF7m78UmBGLDDY6lGOc9TWsfSdJLhiY8,16264
 pybgpkitstream/bgpstreamconfig.py,sha256=vIfEN475WDIZ7kGmi3dnj_1GIQE_r6qkDFETZmMvH5E,6199
-pybgpkitstream/cli.py,sha256=E0E1hO0fzGhy1skBopRufdewsiSy_mA-J8Gf2WxBRxo,4214
+pybgpkitstream/cli.py,sha256=4F5yCNW6OcQFJAj2Hp0rBrDmUDkR1O9x-_aKhVpXrL4,4641
 pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pybgpkitstream/utils.py,sha256=6FwEEpBtY_20BDlJPOPFmTYQGqw7fCBLjXmnd7gjBdQ,404
-pybgpkitstream-0.2.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-pybgpkitstream-0.2.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
-pybgpkitstream-0.2.0.dist-info/METADATA,sha256=EBkydfiRKhgjAz4NotQqix2wX9uSHLK3iDQ_1oUSEqU,2953
-pybgpkitstream-0.2.0.dist-info/RECORD,,
+pybgpkitstream-0.3.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+pybgpkitstream-0.3.0.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
+pybgpkitstream-0.3.0.dist-info/METADATA,sha256=kzkTcOUY8tHeZxIqbCo4pRrfQ3z3ZIzWvAbNZ5ULlfM,2356
+pybgpkitstream-0.3.0.dist-info/RECORD,,

pybgpkitstream-0.2.0.dist-info/METADATA

@@ -1,79 +0,0 @@
-Metadata-Version: 2.3
-Name: pybgpkitstream
-Version: 0.2.0
-Summary: Drop-in replacement for PyBGPStream using BGPKIT
-Author: JustinLoye
-Author-email: JustinLoye <jloye@iij.ad.jp>
-Requires-Dist: aiohttp>=3.12.15
-Requires-Dist: pybgpkit>=0.6.2
-Requires-Dist: pydantic>=2.11.9
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-
-# PyBGPKITStream
-
-A drop-in replacement for PyBGPStream using BGPKIT
-
-## Features
-
-- Effortless BGP Stream Switching:
-  - Seamless, drop-in replacement ([example](tests/test_stream.py#L38))
-  - Lazy message generation: generates time-ordered BGP messages on the fly, consuming minimal memory and making it suitable for large datasets
-  - Supports multiple route collectors
-  - Supports both ribs and updates
-- Caching with concurrent downloading is enabled and is fully compatible with the BGPKIT parser's caching functionality.
-- [Similar performance to PyBGPStream](examples/perf.ipynb)
-- A CLI tool
-
-## Quick start
-
-Installation:
-
-```sh
-pip install pybgpkitstream
-```
-
-Usage:
-
-```python
-import datetime
-from pybgpkitstream import BGPStreamConfig, BGPKITStream
-
-config = BGPStreamConfig(
-    start_time=datetime.datetime(2010, 9, 1, 0, 0),
-    end_time=datetime.datetime(2010, 9, 1, 1, 59),
-    collectors=["route-views.sydney", "route-views.wide"],
-    data_types=["ribs", "updates"],
-)
-
-stream = BGPKITStream.from_config(config)
-
-n_elems = 0
-for _ in stream:
-    n_elems += 1
-    
-print(f"Processed {n_elems} BGP elements")
-```
-
-or in the terminal:
-
-```sh
-pybgpkitstream --start-time 2010-09-01T00:00:00 --end-time 2010-09-01T01:59:00 --collectors route-views.sydney route-views.wide --data-types updates > updates.txt
-```
-
-## Motivation
-
-PyBGPStream is great but the implementation is complex and stops working when UC San Diego experiences a power outage.
-BGPKIT broker and parser are great, but cannot be used to create an ordered stream of BGP messages from multiple collectors and multiple data types.
-
-## Missing features
-
-- live mode
-- `pybgpkitstream.BGPElement` is not fully compatible with `pybgpstream.BGPElem`: missing record_type (BGPKIT limitation), project (BGPKIT limitation), router (could be improved), router_ip (could be improved)
-- CLI output is not yet compatible with `bgpdump -m` or `bgpreader` (right now a similar-looking output is produced)
-
-## Issues
-
-- Program will crash when working with many update files per collector (~ more than few hours of data), only when caching is disabled. This might be caused by [BGPKIT parser not being lazy](https://github.com/bgpkit/bgpkit-parser/pull/239). See [details and workaround fix](examples/many_updates.ipynb)
-- Filters are designed with BGPKIT in mind, and can slightly differ to pybgpstream. See [this file](tests/pybgpstream_utils.py) for a conversion to PyBGPStream filter. Note that for now the filters have not been heavily tested...
-- ... just like the rest of the project. Use at your own risk. The only tests I did are in /tests