pybgpkitstream 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

pybgpkitstream/bgpkitstream.py

@@ -47,12 +47,14 @@ def crc32(input_str: str):
 class BGPKITStream:
     def __init__(
         self,
-        ts_start: datetime.datetime,
-        ts_end: datetime.datetime,
+        ts_start: float,
+        ts_end: float,
         collector_id: str,
         data_type: list[Literal["update", "rib"]],
         cache_dir: str | None,
         filters: dict = {},
+        max_concurrent_downloads: int = 10,
+        chunk_time: float | None = datetime.timedelta(hours=2).seconds,
     ):
         self.ts_start = ts_start
         self.ts_end = ts_end
@@ -60,6 +62,8 @@ class BGPKITStream:
         self.data_type = data_type
         self.cache_dir = cache_dir
         self.filters = filters
+        self.max_concurrent_downloads = max_concurrent_downloads
+        self.chunk_time = chunk_time
 
         self.broker = bgpkit.Broker()
 
@@ -129,8 +133,7 @@ class BGPKITStream:
         self.paths = {"rib": defaultdict(list), "update": defaultdict(list)}
         tasks = []
 
-        CONCURRENT_DOWNLOADS = 10
-        semaphore = asyncio.Semaphore(CONCURRENT_DOWNLOADS)
+        semaphore = asyncio.Semaphore(self.max_concurrent_downloads)
 
         conn = aiohttp.TCPConnector()
         async with aiohttp.ClientSession(connector=conn) as session:
@@ -154,7 +157,7 @@ class BGPKITStream:
 
             if tasks:
                 logging.info(
-                    f"Starting download of {len(tasks)} files with a concurrency of {CONCURRENT_DOWNLOADS}..."
+                    f"Starting download of {len(tasks)} files with a concurrency of {self.max_concurrent_downloads}..."
                 )
                 results = await asyncio.gather(*tasks)
 
@@ -170,6 +173,35 @@ class BGPKITStream:
         return ((elem.timestamp, elem, is_rib, collector) for elem in iterator)
 
     def __iter__(self) -> Iterator[BGPElement]:
+        # Manager mode: spawn smaller worker streams to balance fetch/parse
+        if self.chunk_time:
+            current = self.ts_start
+
+            while current < self.ts_end:
+                chunk_end = min(current + self.chunk_time, self.ts_end)
+
+                logging.info(
+                    f"Processing chunk: {datetime.datetime.fromtimestamp(current)} "
+                    f"to {datetime.datetime.fromtimestamp(chunk_end)}"
+                )
+
+                worker = type(self)(
+                    ts_start=current,
+                    ts_end=chunk_end
+                    - 1,  # remove one second because BGPKIT include border
+                    collector_id=self.collector_id,
+                    data_type=self.data_type,
+                    cache_dir=self.cache_dir,
+                    filters=self.filters,
+                    max_concurrent_downloads=self.max_concurrent_downloads,
+                    chunk_time=None,  # Worker doesn't chunk itself
+                )
+
+                yield from worker
+                current = chunk_end + 1e-7
+
+            return
+
         self._set_urls()
 
         if self.cache_dir:
@@ -225,4 +257,8 @@ class BGPKITStream:
             filters=config.filters.model_dump(exclude_unset=True)
             if config.filters
             else {},
+            max_concurrent_downloads=config.max_concurrent_downloads
+            if config.max_concurrent_downloads
+            else 10,
+            chunk_time=config.chunk_time.seconds if config.chunk_time else None,
         )

pybgpkitstream/bgpstreamconfig.py

@@ -63,3 +63,10 @@ class BGPStreamConfig(BaseModel):
         description="Specifies the directory for caching downloaded files.",
     )
     filters: FilterOptions | None = Field(default=None, description="Optional filters")
+    max_concurrent_downloads: int | None = Field(
+        default=None, description="Maximum concurrent downloads when caching"
+    )
+    chunk_time: datetime.timedelta | None = Field(
+        default=datetime.timedelta(hours=2),
+        description="Interval for the fetch/parse cycle (avoid long prefetch time)",
+    )

{pybgpkitstream-0.1.1.dist-info → pybgpkitstream-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybgpkitstream
-Version: 0.1.1
+Version: 0.1.3
 Summary: Drop-in replacement for PyBGPStream using BGPKIT
 Author: JustinLoye
 Author-email: JustinLoye <jloye@iij.ad.jp>

pybgpkitstream-0.1.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+pybgpkitstream/__init__.py,sha256=kNfv6bvDkaGKjlw0pr9LWVqOQtIGmIPk-VG1ZCBuA38,163
+pybgpkitstream/bgpelement.py,sha256=7mXSUmWThhIbKy2JVsLchoteve0BshT3uH8cdbAe0Go,1176
+pybgpkitstream/bgpkitstream.py,sha256=wJs6c1hs3oZJuPEg4l06MfU8Kz47NmhPngprp3DQ2ws,10100
+pybgpkitstream/bgpstreamconfig.py,sha256=_PHoNhq8lw4QzNKya-KQFQ24dEbTjTkmefFhx0t6K8Q,2873
+pybgpkitstream/cli.py,sha256=E0E1hO0fzGhy1skBopRufdewsiSy_mA-J8Gf2WxBRxo,4214
+pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pybgpkitstream-0.1.3.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+pybgpkitstream-0.1.3.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
+pybgpkitstream-0.1.3.dist-info/METADATA,sha256=M-5bvt3zjpA3IaoqFfiIzdWGgMEfHo16J4u730SRu8Q,2953
+pybgpkitstream-0.1.3.dist-info/RECORD,,

{pybgpkitstream-0.1.1.dist-info → pybgpkitstream-0.1.3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.8.22
+Generator: uv 0.8.24
 Root-Is-Purelib: true
 Tag: py3-none-any

pybgpkitstream-0.1.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-pybgpkitstream/__init__.py,sha256=kNfv6bvDkaGKjlw0pr9LWVqOQtIGmIPk-VG1ZCBuA38,163
-pybgpkitstream/bgpelement.py,sha256=7mXSUmWThhIbKy2JVsLchoteve0BshT3uH8cdbAe0Go,1176
-pybgpkitstream/bgpkitstream.py,sha256=pfjfZOlj0Fs1IDKA254sGNEKfFIho55bvxKksPIa-mA,8609
-pybgpkitstream/bgpstreamconfig.py,sha256=HD8R6JbVEv9-jBB5LOOCfvaBpPZOfTmBlT3UIXzBX8E,2552
-pybgpkitstream/cli.py,sha256=E0E1hO0fzGhy1skBopRufdewsiSy_mA-J8Gf2WxBRxo,4214
-pybgpkitstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pybgpkitstream-0.1.1.dist-info/WHEEL,sha256=-neZj6nU9KAMg2CnCY6T3w8J53nx1kFGw_9HfoSzM60,79
-pybgpkitstream-0.1.1.dist-info/entry_points.txt,sha256=aWhImGlXLtRKkfyJHudcbSp5K5As4ZGL8wXZC0y6q4o,60
-pybgpkitstream-0.1.1.dist-info/METADATA,sha256=yXhpDEQrepf-YDeXM657WZH48ioSTamRyniZP0EqwNE,2953
-pybgpkitstream-0.1.1.dist-info/RECORD,,