corp-extractor 0.4.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

statement_extractor/plugins/qualifiers/sec_edgar.py

@@ -0,0 +1,209 @@
+"""
+SECEdgarQualifierPlugin - Qualifies US ORG entities with SEC data.
+
+DEPRECATED: Use EmbeddingCompanyQualifier instead, which uses a local
+embedding database with pre-loaded SEC Edgar data for faster, offline matching.
+
+Uses the SEC EDGAR API to:
+- Look up CIK (Central Index Key) by company name
+- Retrieve ticker symbol, exchange, filing history
+"""
+
+import logging
+import warnings
+from typing import Optional
+
+from ..base import BaseQualifierPlugin, PluginCapability
+from ...pipeline.context import PipelineContext
+from ...models import ExtractedEntity, EntityQualifiers, EntityType
+
+logger = logging.getLogger(__name__)
+
+# SEC EDGAR API endpoints
+SEC_COMPANY_SEARCH = "https://efts.sec.gov/LATEST/search-index"
+SEC_COMPANY_TICKERS = "https://www.sec.gov/files/company_tickers.json"
+
+
+# DEPRECATED: Not auto-registered. Use EmbeddingCompanyQualifier instead.
+class SECEdgarQualifierPlugin(BaseQualifierPlugin):
+    """
+    DEPRECATED: Use EmbeddingCompanyQualifier instead.
+
+    Qualifier plugin for US ORG entities using SEC EDGAR.
+    Provides CIK and ticker symbol for publicly traded US companies.
+    """
+
+    def __init__(
+        self,
+        timeout: int = 10,
+        cache_results: bool = True,
+    ):
+        """
+        Initialize the SEC EDGAR qualifier.
+
+        DEPRECATED: Use EmbeddingCompanyQualifier instead.
+
+        Args:
+            timeout: API request timeout in seconds
+            cache_results: Whether to cache API results
+        """
+        warnings.warn(
+            "SECEdgarQualifierPlugin is deprecated. Use EmbeddingCompanyQualifier instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self._timeout = timeout
+        self._cache_results = cache_results
+        self._cache: dict[str, Optional[dict]] = {}
+        self._ticker_cache: Optional[dict] = None
+
+    @property
+    def name(self) -> str:
+        return "sec_edgar_qualifier"
+
+    @property
+    def priority(self) -> int:
+        return 30  # Run after GLEIF and Companies House
+
+    @property
+    def capabilities(self) -> PluginCapability:
+        return PluginCapability.EXTERNAL_API | PluginCapability.CACHING
+
+    @property
+    def description(self) -> str:
+        return "Looks up SEC CIK and ticker for US public companies"
+
+    @property
+    def supported_entity_types(self) -> set[EntityType]:
+        return {EntityType.ORG}
+
+    @property
+    def supported_identifier_types(self) -> list[str]:
+        return ["sec_cik", "ticker"]  # Can lookup by CIK or ticker
+
+    @property
+    def provided_identifier_types(self) -> list[str]:
+        return ["sec_cik", "ticker"]  # Provides CIK and ticker
+
+    def qualify(
+        self,
+        entity: ExtractedEntity,
+        context: PipelineContext,
+    ) -> Optional[EntityQualifiers]:
+        """
+        Qualify an ORG entity with SEC EDGAR data.
+
+        Args:
+            entity: The ORG entity to qualify
+            context: Pipeline context
+
+        Returns:
+            EntityQualifiers with CIK and ticker, or None if not found
+        """
+        if entity.type != EntityType.ORG:
+            return None
+
+        # Check cache first
+        cache_key = entity.text.lower().strip()
+        if self._cache_results and cache_key in self._cache:
+            cached = self._cache[cache_key]
+            if cached is None:
+                return None
+            return self._data_to_qualifiers(cached)
+
+        # Search SEC
+        result = self._search_sec(entity.text)
+
+        # Cache result
+        if self._cache_results:
+            self._cache[cache_key] = result
+
+        if result:
+            return self._data_to_qualifiers(result)
+
+        return None
+
+    def _load_ticker_cache(self) -> dict:
+        """Load the SEC company tickers JSON (cached)."""
+        if self._ticker_cache is not None:
+            return self._ticker_cache
+
+        try:
+            import requests
+
+            response = requests.get(SEC_COMPANY_TICKERS, timeout=self._timeout)
+            response.raise_for_status()
+            data = response.json()
+
+            # Build lookup by company name (lowercase)
+            self._ticker_cache = {}
+            for key, company in data.items():
+                name = company.get("title", "").lower()
+                if name:
+                    self._ticker_cache[name] = {
+                        "cik": str(company.get("cik_str", "")),
+                        "ticker": company.get("ticker", ""),
+                        "title": company.get("title", ""),
+                    }
+
+            logger.debug(f"Loaded {len(self._ticker_cache)} SEC company tickers")
+            return self._ticker_cache
+
+        except Exception as e:
+            logger.debug(f"Failed to load SEC ticker cache: {e}")
+            self._ticker_cache = {}
+            return self._ticker_cache
+
+    def _search_sec(self, org_name: str) -> Optional[dict]:
+        """Search SEC for company information."""
+        try:
+            # Load ticker cache
+            ticker_cache = self._load_ticker_cache()
+
+            # Try exact match first
+            org_lower = org_name.lower().strip()
+            if org_lower in ticker_cache:
+                return ticker_cache[org_lower]
+
+            # Try partial match
+            for name, data in ticker_cache.items():
+                if org_lower in name or name in org_lower:
+                    return data
+
+            # Try matching without common suffixes
+            clean_name = org_lower
+            for suffix in [" inc", " inc.", " corp", " corp.", " co", " co.", " ltd", " llc"]:
+                clean_name = clean_name.replace(suffix, "")
+            clean_name = clean_name.strip()
+
+            for name, data in ticker_cache.items():
+                clean_cached = name
+                for suffix in [" inc", " inc.", " corp", " corp.", " co", " co.", " ltd", " llc"]:
+                    clean_cached = clean_cached.replace(suffix, "")
+                clean_cached = clean_cached.strip()
+
+                if clean_name == clean_cached or clean_name in clean_cached or clean_cached in clean_name:
+                    return data
+
+        except Exception as e:
+            logger.debug(f"SEC search error: {e}")
+
+        return None
+
+    def _data_to_qualifiers(self, data: dict) -> EntityQualifiers:
+        """Convert SEC data to EntityQualifiers."""
+        identifiers = {}
+        if data.get("cik"):
+            identifiers["sec_cik"] = data["cik"]
+        if data.get("ticker"):
+            identifiers["ticker"] = data["ticker"]
+
+        return EntityQualifiers(
+            jurisdiction="US",
+            country="US",
+            identifiers=identifiers,
+        )
+
+
+# Allow importing without decorator for testing
+SECEdgarQualifierPluginClass = SECEdgarQualifierPlugin

statement_extractor/plugins/scrapers/__init__.py

@@ -0,0 +1,10 @@
+"""
+Scraper plugins for fetching content from URLs.
+
+Built-in scrapers:
+- http_scraper: Default HTTP scraper using httpx with retries
+"""
+
+from .http import HttpScraperPlugin
+
+__all__ = ["HttpScraperPlugin"]

statement_extractor/plugins/scrapers/http.py

@@ -0,0 +1,236 @@
+"""
+HTTP scraper plugin for fetching web content.
+
+Uses httpx for async HTTP requests with retries, timeouts, and CAPTCHA detection.
+"""
+
+import logging
+from typing import Optional
+
+from ..base import BaseScraperPlugin, ContentType, ScraperResult
+from ...pipeline.registry import PluginRegistry
+
+logger = logging.getLogger(__name__)
+
+
+@PluginRegistry.scraper
+class HttpScraperPlugin(BaseScraperPlugin):
+    """
+    Default HTTP scraper using httpx with retries and timeouts.
+
+    Features:
+    - Async HTTP requests with httpx
+    - Automatic redirect following
+    - Content type detection from headers and URL
+    - CAPTCHA page detection
+    - Configurable timeout and retries
+    """
+
+    def __init__(
+        self,
+        timeout: float = 30.0,
+        max_retries: int = 3,
+        user_agent: str = "Mozilla/5.0 (compatible; StatementExtractor/1.0; +https://github.com/corp-o-rate/statement-extractor)",
+        follow_redirects: bool = True,
+    ):
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._user_agent = user_agent
+        self._follow_redirects = follow_redirects
+
+    @property
+    def name(self) -> str:
+        return "http_scraper"
+
+    @property
+    def priority(self) -> int:
+        return 100  # Default scraper
+
+    @property
+    def description(self) -> str:
+        return "Default HTTP scraper using httpx with retries and CAPTCHA detection"
+
+    async def fetch(self, url: str, timeout: Optional[float] = None) -> ScraperResult:
+        """
+        Fetch content from a URL with retries and CAPTCHA detection.
+
+        Args:
+            url: The URL to fetch
+            timeout: Request timeout in seconds (uses instance default if None)
+
+        Returns:
+            ScraperResult with content, content type, and any errors
+        """
+        import httpx
+
+        timeout = timeout or self._timeout
+        last_error: Optional[str] = None
+
+        for attempt in range(self._max_retries):
+            try:
+                async with httpx.AsyncClient(
+                    timeout=timeout,
+                    follow_redirects=self._follow_redirects,
+                ) as client:
+                    logger.debug(f"Fetching URL: {url} (attempt {attempt + 1})")
+
+                    response = await client.get(
+                        url,
+                        headers={"User-Agent": self._user_agent},
+                    )
+
+                    content_type = self._detect_content_type(
+                        dict(response.headers), url
+                    )
+
+                    # Check for CAPTCHA if HTML
+                    error = None
+                    if content_type == ContentType.HTML:
+                        if self._is_captcha_page(response.content):
+                            error = "CAPTCHA or challenge page detected"
+                            logger.warning(f"CAPTCHA detected at {url}")
+
+                    return ScraperResult(
+                        url=url,
+                        final_url=str(response.url),
+                        content=response.content,
+                        content_type=content_type,
+                        headers=dict(response.headers),
+                        error=error,
+                    )
+
+            except httpx.TimeoutException as e:
+                last_error = f"Request timed out after {timeout}s"
+                logger.warning(f"Timeout fetching {url}: {e}")
+            except httpx.ConnectError as e:
+                last_error = f"Connection error: {e}"
+                logger.warning(f"Connection error fetching {url}: {e}")
+            except httpx.HTTPStatusError as e:
+                last_error = f"HTTP {e.response.status_code}: {e.response.reason_phrase}"
+                logger.warning(f"HTTP error fetching {url}: {e}")
+                # Don't retry on 4xx errors
+                if 400 <= e.response.status_code < 500:
+                    break
+            except Exception as e:
+                last_error = f"Unexpected error: {e}"
+                logger.exception(f"Error fetching {url}")
+
+        # All retries failed
+        return ScraperResult(
+            url=url,
+            final_url=url,
+            content=b"",
+            content_type=ContentType.UNKNOWN,
+            error=last_error or "Unknown error",
+        )
+
+    async def head(self, url: str, timeout: Optional[float] = None) -> ScraperResult:
+        """
+        Check content type without downloading the full body.
+
+        Args:
+            url: The URL to check
+            timeout: Request timeout in seconds
+
+        Returns:
+            ScraperResult with content_type populated (content is empty)
+        """
+        import httpx
+
+        timeout = timeout or self._timeout
+
+        try:
+            async with httpx.AsyncClient(
+                timeout=timeout,
+                follow_redirects=self._follow_redirects,
+            ) as client:
+                response = await client.head(
+                    url,
+                    headers={"User-Agent": self._user_agent},
+                )
+
+                content_type = self._detect_content_type(
+                    dict(response.headers), url
+                )
+
+                return ScraperResult(
+                    url=url,
+                    final_url=str(response.url),
+                    content=b"",
+                    content_type=content_type,
+                    headers=dict(response.headers),
+                )
+
+        except Exception as e:
+            logger.warning(f"HEAD request failed for {url}: {e}")
+            # Fall back to full fetch
+            return await self.fetch(url, timeout)
+
+    @staticmethod
+    def _detect_content_type(headers: dict[str, str], url: str) -> ContentType:
+        """
+        Detect content type from HTTP headers and URL.
+
+        Priority:
+        1. Content-Type header
+        2. URL file extension
+        """
+        content_type_header = headers.get("content-type", "").lower()
+
+        # Check Content-Type header
+        if "application/pdf" in content_type_header:
+            return ContentType.PDF
+        if any(mime in content_type_header for mime in [
+            "text/html",
+            "application/xhtml+xml",
+        ]):
+            return ContentType.HTML
+
+        # Check URL extension
+        url_lower = url.lower().split("?")[0]  # Remove query params
+        if url_lower.endswith(".pdf"):
+            return ContentType.PDF
+        if url_lower.endswith((".html", ".htm")):
+            return ContentType.HTML
+
+        # Default based on content-type
+        if content_type_header.startswith("text/"):
+            return ContentType.HTML
+        if content_type_header.startswith(("image/", "audio/", "video/")):
+            return ContentType.BINARY
+
+        return ContentType.UNKNOWN
+
+    @staticmethod
+    def _is_captcha_page(content: bytes) -> bool:
+        """
+        Detect CAPTCHA or challenge pages.
+
+        Checks for common CAPTCHA patterns in HTML content.
+        """
+        try:
+            html = content.decode("utf-8", errors="replace").lower()
+        except Exception:
+            return False
+
+        # Only check small pages (challenge pages are usually small)
+        if len(html) > 50000:
+            return False
+
+        # Common CAPTCHA/challenge indicators
+        captcha_patterns = [
+            "captcha",
+            "cloudflare",
+            "checking your browser",
+            "please verify you are a human",
+            "access denied",
+            "bot protection",
+            "ddos protection",
+            "just a moment",
+            "enable javascript",
+            "please enable cookies",
+            "verify you are human",
+            "security check",
+        ]
+
+        return any(pattern in html for pattern in captcha_patterns)

statement_extractor/plugins/splitters/__init__.py

@@ -0,0 +1,13 @@
+"""
+Splitter plugins for Stage 1 (Splitting).
+
+Splits text into atomic triples.
+"""
+
+from .base import BaseSplitterPlugin
+from .t5_gemma import T5GemmaSplitter
+
+__all__ = [
+    "BaseSplitterPlugin",
+    "T5GemmaSplitter",
+]

statement_extractor/plugins/splitters/base.py

@@ -0,0 +1,9 @@
+"""
+Base class for splitter plugins.
+
+Re-exports BaseSplitterPlugin from the main plugins module.
+"""
+
+from ..base import BaseSplitterPlugin
+
+__all__ = ["BaseSplitterPlugin"]