datasette-libfec 0.0.1a4__tar.gz → 0.0.1a5__tar.gz

{datasette_libfec-0.0.1a4 → datasette_libfec-0.0.1a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasette-libfec
-Version: 0.0.1a4
+Version: 0.0.1a5
 Author: Alex Garcia
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/datasette/datasette-libfec
@@ -13,7 +13,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: datasette>=1a23
 Requires-Dist: datasette-plugin-router>=0.0.1a2
-Requires-Dist: libfec>=0.0.18
+Requires-Dist: libfec>=0.0.19
 Dynamic: license-file
 
 # datasette-libfec

{datasette_libfec-0.0.1a4 → datasette_libfec-0.0.1a5}/datasette_libfec/__init__.py

@@ -6,7 +6,11 @@ from typing import Optional
 import json
 import os
 
-from .routes import router
+# Import route modules to trigger route registration on the shared router
+# pylint: disable=unused-import
+from . import routes_rss, routes_export, routes_search, routes_exports, routes_pages
+from .router import router
+_ = routes_rss, routes_export, routes_search, routes_exports, routes_pages
 
 # https://vite.dev/guide/backend-integration.html
 class ManifestChunk(BaseModel):

datasette_libfec-0.0.1a5/datasette_libfec/libfec_client.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import os
+import sys
+import asyncio
+import subprocess
+import time
+from pathlib import Path
+from typing import Optional, TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from .libfec_rpc_client import LibfecRpcClient
+    from .libfec_export_rpc_client import LibfecExportRpcClient
+
+
+class LibfecClient:
+    def __init__(self):
+        bin_path = os.environ.get("DATASETTE_LIBFEC_BIN_PATH")
+        if bin_path:
+            self.libfec_path = Path(bin_path)
+        else:
+            self.libfec_path = Path(sys.executable).parent / 'libfec'
+
+    async def _run_libfec_command_async(self, args):
+        """Async command execution - doesn't block event loop"""
+        print(f"Running async: {args}")
+        process = await asyncio.create_subprocess_exec(
+            str(self.libfec_path),
+            *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+
+        if process.returncode != 0:
+            raise Exception(f"libfec error: {stderr.decode()}")
+        return stdout.decode()
+
+    async def export(self, committee_id: str, cycle: int, output_db: str) -> str:
+        """Export FEC data (async - won't block event loop)"""
+        return await self._run_libfec_command_async([
+            'export', committee_id,
+            '--election', str(cycle),
+            '-o', output_db
+          ])
+
+    async def rss_watch(self, output_db: str, state: Optional[str] = None, cover_only: bool = True):
+        """Run single RSS watch command (async - won't block event loop)"""
+        args = ['rss', '--since', '1 day']
+        if cover_only:
+            args.append('--cover-only')
+        args.extend(['-x', output_db])
+        if state:
+            args.extend(['--state', state])
+        return await self._run_libfec_command_async(args)
+
+    async def rss_watch_with_progress(
+        self,
+        output_db: str,
+        state: Optional[str],
+        cover_only: bool,
+        watcher_state: RssWatcherState
+    ) -> None:
+        """
+        Run RSS watch using RPC mode with real-time progress tracking.
+
+        Updates watcher_state with progress information from RPC notifications.
+        """
+        from .libfec_rpc_client import LibfecRpcClient, RpcError
+
+        # Reset progress state
+        watcher_state.phase = "idle"
+        watcher_state.exported_count = 0
+        watcher_state.total_count = 0
+        watcher_state.current_filing_id = None
+        watcher_state.feed_title = None
+        watcher_state.feed_last_modified = None
+        watcher_state.error_message = None
+        watcher_state.error_code = None
+        watcher_state.error_data = None
+        watcher_state.sync_start_time = time.time()
+
+        # Use RPC mode
+        rpc_client = LibfecRpcClient(str(self.libfec_path))
+        watcher_state.rpc_client = rpc_client
+
+        def on_progress(notification: dict) -> None:
+            """Progress callback - updates watcher_state from RPC notifications"""
+            method = notification.get("method")
+            params = notification.get("params", {})
+
+            if method == "sync/progress":
+                watcher_state.phase = params.get("phase", "idle")
+                watcher_state.exported_count = params.get("exported_count", 0)
+                watcher_state.total_count = params.get("total_count", 0)
+                watcher_state.current_filing_id = params.get("current_filing_id")
+                watcher_state.feed_title = params.get("feed_title")
+                watcher_state.feed_last_modified = params.get("feed_last_modified")
+
+                # Update currently_syncing based on phase
+                watcher_state.currently_syncing = watcher_state.phase in (
+                    "fetching", "exporting"
+                )
+
+        try:
+            watcher_state.currently_syncing = True
+            await rpc_client.start_process()
+
+            result = await rpc_client.sync_start(
+                since="1 day",
+                state=state,
+                cover_only=cover_only,
+                output_path=output_db,
+                progress_callback=on_progress
+            )
+
+            # Mark as complete
+            watcher_state.phase = "complete"
+            print(f"RSS sync complete: {result}")
+
+        except RpcError as e:
+            watcher_state.phase = "error"
+            watcher_state.error_message = e.message
+            watcher_state.error_code = e.code
+            watcher_state.error_data = str(e.data) if e.data else None
+            print(f"RPC error: {e}, data: {e.data}")
+
+        except asyncio.TimeoutError:
+            watcher_state.phase = "error"
+            watcher_state.error_message = "Sync timed out after 5 minutes"
+            print("RSS sync timeout")
+
+        except Exception as e:
+            watcher_state.phase = "error"
+            watcher_state.error_message = str(e)
+            print(f"RSS sync error: {e}")
+
+        finally:
+            watcher_state.currently_syncing = False
+            try:
+                await rpc_client.shutdown()
+            except Exception as e:
+                print(f"Error shutting down RPC client: {e}")
+                try:
+                    await rpc_client.terminate()
+                except Exception:
+                    pass
+            watcher_state.rpc_client = None
+
+    async def export_with_progress(
+        self,
+        output_db: str,
+        filings: Optional[List[str]],
+        cycle: Optional[int],
+        cover_only: bool,
+        clobber: bool,
+        export_state: ExportState
+    ) -> None:
+        """
+        Run export using RPC mode with real-time progress tracking.
+
+        Updates export_state with progress information from RPC notifications.
+        """
+        from .libfec_export_rpc_client import LibfecExportRpcClient, RpcError
+
+        # Reset progress state
+        export_state.phase = "idle"
+        export_state.completed = 0
+        export_state.total = 0
+        export_state.current_filing_id = None
+        export_state.current = None
+        export_state.total_exported = None
+        export_state.warnings = []
+        export_state.error_message = None
+        export_state.export_start_time = time.time()
+
+        # Use RPC mode
+        rpc_client = LibfecExportRpcClient(str(self.libfec_path), output_db)
+        export_state.rpc_client = rpc_client
+
+        def on_progress(notification: dict) -> None:
+            """Progress callback - updates export_state from RPC notifications"""
+            method = notification.get("method")
+            params = notification.get("params", {})
+
+            if method == "export/progress":
+                export_state.phase = params.get("phase", "idle")
+                export_state.completed = params.get("completed", 0)
+                export_state.total = params.get("total", 0)
+                export_state.current_filing_id = params.get("current_filing_id")
+                export_state.current = params.get("current")
+                export_state.total_exported = params.get("total_exported")
+                export_state.warnings = params.get("warnings", [])
+                if params.get("error_message"):
+                    export_state.error_message = params["error_message"]
+
+        try:
+            export_state.running = True
+            await rpc_client.start_process()
+
+            result = await rpc_client.export_start(
+                filings=filings,
+                cycle=cycle,
+                cover_only=cover_only,
+                clobber=clobber,
+                progress_callback=on_progress
+            )
+
+            # Mark as complete
+            export_state.phase = "complete"
+            print(f"Export complete: {result}")
+
+        except RpcError as e:
+            export_state.phase = "error"
+            export_state.error_message = str(e.data) if e.data else e.message
+            print(f"RPC error: {e}")
+
+        except asyncio.TimeoutError:
+            export_state.phase = "error"
+            export_state.error_message = "Export timed out after 5 minutes"
+            print("Export timeout")
+
+        except Exception as e:
+            export_state.phase = "error"
+            export_state.error_message = str(e)
+            print(f"Export error: {e}")
+
+        finally:
+            export_state.running = False
+            try:
+                await rpc_client.shutdown()
+            except Exception as e:
+                print(f"Error shutting down RPC client: {e}")
+                try:
+                    await rpc_client.terminate()
+                except Exception:
+                    pass
+            export_state.rpc_client = None
+
+
+# RSS watcher state
+class RssWatcherState:
+    def __init__(self):
+        self.task: Optional[asyncio.Task] = None
+        self.running = False
+        self.interval = 60
+        self.state: Optional[str] = None
+        self.cover_only = True
+        self.output_db: Optional[str] = None
+        self.next_sync_time: Optional[float] = None
+        self.currently_syncing = False
+
+        # Progress tracking fields for RPC mode
+        self.phase: str = "idle"  # idle|fetching|exporting|complete|canceled|error
+        self.exported_count: int = 0
+        self.total_count: int = 0
+        self.current_filing_id: Optional[str] = None
+        self.feed_title: Optional[str] = None
+        self.feed_last_modified: Optional[str] = None
+        self.error_message: Optional[str] = None
+        self.error_code: Optional[int] = None
+        self.error_data: Optional[str] = None
+        self.sync_start_time: Optional[float] = None
+        self.rpc_client: Optional["LibfecRpcClient"] = None
+
+
+# Export state
+class ExportState:
+    def __init__(self):
+        self.export_id: Optional[str] = None
+        self.running = False
+
+        # Progress tracking fields for export RPC mode
+        self.phase: str = "idle"  # idle|sourcing|downloading_bulk|exporting|complete|canceled|error
+        self.completed: int = 0
+        self.total: int = 0
+        self.current_filing_id: Optional[str] = None
+        self.current: Optional[str] = None  # For bulk download phase
+        self.total_exported: Optional[int] = None
+        self.warnings: List[str] = []
+        self.error_message: Optional[str] = None
+        self.export_start_time: Optional[float] = None
+        self.rpc_client: Optional["LibfecExportRpcClient"] = None

datasette_libfec-0.0.1a5/datasette_libfec/libfec_export_rpc_client.py

@@ -0,0 +1,358 @@
+"""
+JSON-RPC 2.0 client for libfec export --rpc mode.
+
+Manages libfec export subprocess lifecycle using JSONL protocol over stdin/stdout.
+"""
+
+import asyncio
+import json
+import logging
+from typing import Any, Callable, Optional, List
+
+logger = logging.getLogger(__name__)
+
+
+class RpcError(Exception):
+    """JSON-RPC error response"""
+    def __init__(self, code: int, message: str, data: Any = None):
+        self.code = code
+        self.message = message
+        self.data = data
+        super().__init__(f"RPC Error {code}: {message}")
+
+
+class LibfecExportRpcClient:
+    """
+    Manages libfec export --rpc process lifecycle.
+
+    Uses JSON-RPC 2.0 over JSONL protocol:
+    - Requests/responses: JSON objects with id field
+    - Notifications: JSON objects without id field
+    """
+
+    def __init__(self, libfec_path: str, output_db: str):
+        self.libfec_path = libfec_path
+        self.output_db = output_db
+        self.process: Optional[asyncio.subprocess.Process] = None
+        self.request_id = 0
+        self.pending_requests: dict[int, asyncio.Future] = {}
+        self.listen_task: Optional[asyncio.Task] = None
+        self.progress_callback: Optional[Callable] = None
+        self.completion_future: Optional[asyncio.Future] = None
+        self.ready_future: Optional[asyncio.Future] = None
+
+    async def start_process(self) -> None:
+        """Spawn libfec export --rpc subprocess"""
+        if self.process is not None:
+            raise RuntimeError("Process already started")
+
+        logger.info(f"Starting libfec export --rpc: {self.libfec_path} -o {self.output_db}")
+
+        # Create ready future to wait for ready notification
+        self.ready_future = asyncio.Future()
+
+        self.process = await asyncio.create_subprocess_exec(
+            self.libfec_path,
+            "export",
+            "--rpc",
+            "-o",
+            self.output_db,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+        # Start listening for messages
+        self.listen_task = asyncio.create_task(self._listen_for_messages())
+
+        # Wait for ready notification with timeout
+        try:
+            await asyncio.wait_for(self.ready_future, timeout=5.0)
+            logger.info("RPC server ready")
+        except asyncio.TimeoutError:
+            logger.error("Timeout waiting for ready notification")
+            await self.terminate()
+            raise RuntimeError("libfec process did not send ready notification")
+
+    async def _listen_for_messages(self) -> None:
+        """
+        Read stdout line-by-line (JSONL protocol).
+        Route responses to waiting futures by request ID.
+        Route notifications to progress_callback.
+        """
+        if not self.process or not self.process.stdout:
+            return
+
+        try:
+            while True:
+                line = await self.process.stdout.readline()
+                if not line:
+                    # EOF - process died
+                    logger.error("libfec process stdout EOF (process crashed?)")
+                    # Cancel all pending requests
+                    for future in self.pending_requests.values():
+                        if not future.done():
+                            future.set_exception(
+                                RuntimeError("libfec process terminated unexpectedly")
+                            )
+                    # Cancel completion future if waiting
+                    if self.completion_future and not self.completion_future.done():
+                        self.completion_future.set_exception(
+                            RuntimeError("libfec process terminated before export completed")
+                        )
+                    break
+
+                try:
+                    msg = json.loads(line.decode())
+                except json.JSONDecodeError as e:
+                    logger.error(f"Invalid JSON from libfec: {line!r} - {e}")
+                    continue
+
+                # Check if response or notification
+                if "id" in msg:
+                    # Response - match to pending request
+                    request_id = msg["id"]
+                    future = self.pending_requests.get(request_id)
+
+                    if future and not future.done():
+                        if "error" in msg:
+                            err = msg["error"]
+                            future.set_exception(
+                                RpcError(
+                                    err.get("code", -1),
+                                    err.get("message", "Unknown error"),
+                                    err.get("data")
+                                )
+                            )
+                        elif "result" in msg:
+                            future.set_result(msg["result"])
+                        else:
+                            future.set_exception(
+                                RuntimeError(f"Invalid RPC response: {msg}")
+                            )
+                else:
+                    # Notification - deliver to callback
+                    if "method" in msg:
+                        # Check if this is a ready notification
+                        if msg.get("method") == "ready":
+                            if self.ready_future and not self.ready_future.done():
+                                self.ready_future.set_result(msg.get("params", {}))
+
+                        # Check if this is a completion notification
+                        elif msg.get("method") == "export/progress":
+                            params = msg.get("params", {})
+                            phase = params.get("phase")
+
+                            # Resolve completion future on terminal phases
+                            if phase in ("complete", "canceled", "error"):
+                                if self.completion_future and not self.completion_future.done():
+                                    self.completion_future.set_result(params)
+
+                        # Deliver to callback
+                        if self.progress_callback:
+                            try:
+                                self.progress_callback(msg)
+                            except Exception as e:
+                                logger.error(f"Progress callback error: {e}", exc_info=True)
+
+        except asyncio.CancelledError:
+            logger.debug("Message listener cancelled")
+            raise
+        except Exception as e:
+            logger.error(f"Error in message listener: {e}", exc_info=True)
+
+    async def send_request(
+        self,
+        method: str,
+        params: Optional[dict] = None,
+        timeout: float = 5.0
+    ) -> Any:
+        """
+        Send JSON-RPC request via stdin, wait for response.
+
+        Args:
+            method: RPC method name
+            params: Method parameters
+            timeout: Response timeout in seconds
+
+        Returns:
+            Result from response
+
+        Raises:
+            RpcError: On JSON-RPC error response
+            TimeoutError: On timeout
+            RuntimeError: On process errors
+        """
+        if not self.process or not self.process.stdin:
+            raise RuntimeError("Process not started")
+
+        self.request_id += 1
+        request = {
+            "jsonrpc": "2.0",
+            "id": self.request_id,
+            "method": method,
+        }
+        if params is not None:
+            request["params"] = params
+
+        # Create future for response
+        future: asyncio.Future = asyncio.Future()
+        self.pending_requests[self.request_id] = future
+
+        try:
+            # Send request
+            request_json = json.dumps(request) + "\n"
+            self.process.stdin.write(request_json.encode())
+            await self.process.stdin.drain()
+
+            logger.debug(f"Sent RPC request: {method} (id={self.request_id})")
+
+            # Wait for response with timeout
+            result = await asyncio.wait_for(future, timeout=timeout)
+            return result
+
+        except asyncio.TimeoutError:
+            logger.error(f"RPC request timeout: {method}")
+            raise TimeoutError(f"RPC request timed out: {method}")
+        finally:
+            # Clean up pending request
+            self.pending_requests.pop(self.request_id, None)
+
+    async def export_start(
+        self,
+        filings: Optional[List[str]],
+        cycle: Optional[int],
+        cover_only: bool,
+        clobber: bool,
+        progress_callback: Callable,
+        write_metadata: bool = True,
+    ) -> dict:
+        """
+        Start export operation with progress tracking.
+
+        Args:
+            filings: Filing IDs, committee IDs, or candidate IDs to export
+            cycle: Election cycle year (e.g., 2024)
+            cover_only: Only export cover pages
+            clobber: Overwrite existing database content
+            progress_callback: Called with export/progress notifications
+            write_metadata: Whether to write metadata tables in output database
+
+        Returns:
+            Final export result
+
+        Raises:
+            RpcError: On export errors
+            TimeoutError: On export timeout (300s)
+        """
+        self.progress_callback = progress_callback
+
+        params = {}
+        if filings is not None:
+            params["filings"] = filings
+        if cycle is not None:
+            params["cycle"] = cycle
+        if cover_only:
+            params["cover_only"] = cover_only
+        if clobber:
+            params["clobber"] = clobber
+        if write_metadata:
+            params["write_metadata"] = write_metadata
+        
+        # Create completion future to wait for final notification
+        self.completion_future = asyncio.Future()
+
+        # Send export/start request (just starts the export)
+        logger.info(f"Sending export/start with params: {params}")
+        start_result = await self.send_request("export/start", params, timeout=30.0)
+        logger.info(f"Export started: {start_result}")
+
+        # Keep sending export/status requests to keep the RPC server processing exports
+        async def poll_status():
+            while self.completion_future and not self.completion_future.done():
+                try:
+                    await asyncio.sleep(0.5)  # Poll every 500ms
+                    if self.completion_future and not self.completion_future.done():
+                        await self.send_request("export/status", timeout=5.0)
+                except Exception as e:
+                    logger.debug(f"Status poll error (expected on completion): {e}")
+                    break
+
+        # Start status polling task
+        poll_task = asyncio.create_task(poll_status())
+
+        # Wait for completion notification with 300s timeout
+        try:
+            completion_result = await asyncio.wait_for(self.completion_future, timeout=300.0)
+
+            # Check if export completed with error
+            if completion_result.get("phase") == "error":
+                error_msg = completion_result.get("error_message", "Unknown error")
+                error_code = completion_result.get("error_code", -1)
+                error_data = completion_result.get("error_data")
+                raise RpcError(error_code, error_msg, error_data)
+
+            return completion_result
+        except asyncio.TimeoutError:
+            logger.error("Export did not complete within 300 seconds")
+            raise TimeoutError("Export timed out after 5 minutes")
+        finally:
+            # Cancel polling task
+            poll_task.cancel()
+            try:
+                await poll_task
+            except asyncio.CancelledError:
+                pass
+
+    async def export_cancel(self) -> dict:
+        """Cancel in-progress export"""
+        return await self.send_request("export/cancel", timeout=5.0)
+
+    async def shutdown(self) -> None:
+        """Gracefully shutdown RPC process"""
+        if not self.process:
+            return
+
+        try:
+            await self.send_request("shutdown", timeout=2.0)
+        except Exception as e:
+            logger.warning(f"Shutdown request failed: {e}")
+
+        # Wait for process to exit
+        try:
+            await asyncio.wait_for(self.process.wait(), timeout=5.0)
+        except asyncio.TimeoutError:
+            logger.warning("Process did not exit after shutdown, terminating")
+            await self.terminate()
+
+        # Cancel listener task
+        if self.listen_task and not self.listen_task.done():
+            self.listen_task.cancel()
+            try:
+                await self.listen_task
+            except asyncio.CancelledError:
+                pass
+
+        self.process = None
+
+    async def terminate(self) -> None:
+        """Force kill the process (for cleanup)"""
+        if self.process:
+            try:
+                self.process.terminate()
+                await asyncio.wait_for(self.process.wait(), timeout=2.0)
+            except asyncio.TimeoutError:
+                logger.warning("Process did not terminate, killing")
+                self.process.kill()
+                await self.process.wait()
+            except Exception as e:
+                logger.error(f"Error terminating process: {e}")
+
+        if self.listen_task and not self.listen_task.done():
+            self.listen_task.cancel()
+            try:
+                await self.listen_task
+            except asyncio.CancelledError:
+                pass
+
+        self.process = None