corp-extractor 0.5.0__py3-none-any.whl → 0.9.3__py3-none-any.whl

statement_extractor/document/deduplicator.py

@@ -0,0 +1,171 @@
+"""
+StatementDeduplicator - Hash-based deduplication for statements.
+
+Removes duplicate statements across chunks using normalized hashing.
+Works with Stage 2+ output (PipelineStatement, LabeledStatement) which
+have subject-predicate-object structure.
+"""
+
+import hashlib
+import logging
+from typing import TypeVar, Union
+
+from ..models.labels import LabeledStatement
+from ..models.statement import PipelineStatement
+
+logger = logging.getLogger(__name__)
+
+# Type variable for generic deduplication
+T = TypeVar("T", PipelineStatement, LabeledStatement)
+
+
+class StatementDeduplicator:
+    """
+    Deduplicates statements using normalized hash comparison.
+
+    Uses a hash of normalized (subject, predicate, object) to identify
+    duplicates. Keeps the first occurrence of each unique statement.
+
+    Works with PipelineStatement (Stage 2) and LabeledStatement (Stage 4).
+    """
+
+    def __init__(self):
+        """Initialize the deduplicator."""
+        self._seen_hashes: set[str] = set()
+
+    def reset(self) -> None:
+        """Reset the deduplicator state, clearing all seen hashes."""
+        self._seen_hashes.clear()
+        logger.debug("Deduplicator state reset")
+
+    def _normalize_text(self, text: str) -> str:
+        """
+        Normalize text for comparison.
+
+        - Lowercase
+        - Strip whitespace
+        - Collapse multiple spaces
+        """
+        return " ".join(text.lower().strip().split())
+
+    def _get_triple_parts(
+        self,
+        stmt: Union[PipelineStatement, LabeledStatement],
+    ) -> tuple[str, str, str]:
+        """
+        Extract (subject, predicate, object) from a statement.
+
+        Handles different statement types consistently.
+        """
+        if isinstance(stmt, LabeledStatement):
+            return (
+                stmt.statement.subject.text,
+                stmt.statement.predicate,
+                stmt.statement.object.text,
+            )
+        else:
+            # PipelineStatement
+            return (
+                stmt.subject.text,
+                stmt.predicate,
+                stmt.object.text,
+            )
+
+    def _hash_triple(
+        self,
+        stmt: Union[PipelineStatement, LabeledStatement],
+    ) -> str:
+        """
+        Generate a hash for a statement triple.
+
+        Uses normalized text to catch near-duplicates with different
+        casing or whitespace.
+        """
+        subj, pred, obj = self._get_triple_parts(stmt)
+
+        key = (
+            self._normalize_text(subj),
+            self._normalize_text(pred),
+            self._normalize_text(obj),
+        )
+
+        # Use sha256 and truncate to 16 chars for reasonable uniqueness
+        return hashlib.sha256(str(key).encode()).hexdigest()[:16]
+
+    def is_duplicate(
+        self,
+        stmt: Union[PipelineStatement, LabeledStatement],
+    ) -> bool:
+        """
+        Check if a statement is a duplicate.
+
+        Also marks the statement as seen if it's not a duplicate.
+
+        Args:
+            stmt: Statement to check
+
+        Returns:
+            True if this is a duplicate of a previously seen statement
+        """
+        hash_value = self._hash_triple(stmt)
+
+        if hash_value in self._seen_hashes:
+            return True
+
+        self._seen_hashes.add(hash_value)
+        return False
+
+    def filter_duplicates(self, statements: list[T]) -> list[T]:
+        """
+        Filter out duplicate statements from a list.
+
+        Preserves order and keeps the first occurrence of each unique statement.
+
+        Args:
+            statements: List of statements to deduplicate
+
+        Returns:
+            List with duplicates removed
+        """
+        if not statements:
+            return []
+
+        original_count = len(statements)
+        result = []
+
+        for stmt in statements:
+            if not self.is_duplicate(stmt):
+                result.append(stmt)
+
+        removed = original_count - len(result)
+        if removed > 0:
+            logger.info(f"Deduplication removed {removed} statements ({len(result)} remaining)")
+
+        return result
+
+    def deduplicate_batch(
+        self,
+        statements: list[T],
+        reset_first: bool = True,
+    ) -> list[T]:
+        """
+        Deduplicate a batch of statements.
+
+        Optionally resets state before processing to ensure clean deduplication.
+
+        Args:
+            statements: List of statements to deduplicate
+            reset_first: Whether to reset seen hashes before processing
+
+        Returns:
+            Deduplicated list of statements
+        """
+        if reset_first:
+            self.reset()
+
+        return self.filter_duplicates(statements)
+
+    @property
+    def seen_count(self) -> int:
+        """Get the number of unique statements seen."""
+        return len(self._seen_hashes)

statement_extractor/document/html_extractor.py

@@ -0,0 +1,246 @@
+"""
+HTML text extraction utilities.
+
+Extracts clean text content from HTML pages, prioritizing article content
+and removing navigation, headers, footers, and other non-content elements.
+"""
+
+import logging
+import re
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def extract_text_from_html(html: str) -> tuple[str, Optional[str]]:
+    """
+    Extract clean text and title from HTML.
+
+    Removes scripts, styles, navigation, and other non-content elements.
+    Prioritizes article or main content areas.
+
+    Args:
+        html: Raw HTML string
+
+    Returns:
+        Tuple of (extracted_text, title or None)
+    """
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        raise ImportError(
+            "BeautifulSoup is required for HTML extraction. "
+            "Install with: pip install beautifulsoup4"
+        )
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Remove unwanted elements
+    for tag_name in [
+        "script",
+        "style",
+        "nav",
+        "footer",
+        "header",
+        "aside",
+        "noscript",
+        "iframe",
+        "form",
+        "button",
+        "input",
+        "select",
+        "textarea",
+    ]:
+        for tag in soup.find_all(tag_name):
+            tag.decompose()
+
+    # Remove elements with common non-content class/id patterns
+    non_content_patterns = [
+        "nav",
+        "menu",
+        "sidebar",
+        "footer",
+        "header",
+        "comment",
+        "advertisement",
+        "ad-",
+        "social",
+        "share",
+        "related",
+        "recommended",
+        "popup",
+        "modal",
+        "cookie",
+        "banner",
+        "promo",
+    ]
+
+    # Collect elements to remove first, then decompose
+    # (decomposing while iterating can cause issues)
+    elements_to_remove = []
+
+    for element in soup.find_all(class_=True):
+        if element.attrs is None:
+            continue
+        classes = element.get("class", [])
+        if classes:
+            class_str = " ".join(classes).lower()
+            if any(pattern in class_str for pattern in non_content_patterns):
+                elements_to_remove.append(element)
+
+    for element in soup.find_all(id=True):
+        if element.attrs is None:
+            continue
+        element_id = element.get("id", "")
+        if element_id and any(pattern in element_id.lower() for pattern in non_content_patterns):
+            elements_to_remove.append(element)
+
+    for element in elements_to_remove:
+        try:
+            element.decompose()
+        except Exception:
+            pass  # Element may already be decomposed
+
+    # Get title
+    title = None
+    if soup.title and soup.title.string:
+        title = soup.title.string.strip()
+        # Clean up common title patterns (e.g., "Article Title | Site Name")
+        title = re.split(r"\s*[|—\-]\s*", title)[0].strip()
+
+    # Find main content area
+    content = None
+
+    # Priority: article > main > [role="main"] > body
+    for selector in ["article", "main", "[role='main']", ".content", "#content"]:
+        content = soup.select_one(selector)
+        if content and len(content.get_text(strip=True)) > 100:
+            break
+
+    if not content:
+        content = soup.body or soup
+
+    # Extract text using BeautifulSoup's get_text with newline separator
+    text = content.get_text(separator="\n", strip=True)
+
+    # Clean up whitespace
+    text = _clean_whitespace(text)
+
+    logger.debug(f"Extracted {len(text)} chars from HTML (title: {title})")
+
+    return text, title
+
+
+def _clean_whitespace(text: str) -> str:
+    """
+    Clean up whitespace while preserving paragraph structure.
+
+    Args:
+        text: Raw extracted text
+
+    Returns:
+        Cleaned text
+    """
+    # Normalize line breaks
+    text = re.sub(r"\r\n?", "\n", text)
+
+    # Collapse multiple spaces (but not newlines)
+    text = re.sub(r"[^\S\n]+", " ", text)
+
+    # Collapse multiple newlines to max 2
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    # Remove leading/trailing whitespace from each line
+    lines = [line.strip() for line in text.split("\n")]
+    text = "\n".join(lines)
+
+    # Remove empty lines at start/end
+    text = text.strip()
+
+    return text
+
+
+def extract_article_content(html: str) -> tuple[str, dict]:
+    """
+    Extract article content with metadata.
+
+    Attempts to extract structured article data including:
+    - Title
+    - Author
+    - Published date
+    - Main content
+
+    Args:
+        html: Raw HTML string
+
+    Returns:
+        Tuple of (content, metadata dict)
+    """
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        raise ImportError(
+            "BeautifulSoup is required for HTML extraction. "
+            "Install with: pip install beautifulsoup4"
+        )
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    metadata = {}
+
+    # Extract title
+    title = None
+    # Try og:title first
+    og_title = soup.find("meta", property="og:title")
+    if og_title and og_title.get("content"):
+        title = og_title["content"].strip()
+    elif soup.title and soup.title.string:
+        title = soup.title.string.strip()
+
+    if title:
+        metadata["title"] = title
+
+    # Extract author
+    author = None
+    author_meta = soup.find("meta", attrs={"name": "author"})
+    if author_meta and author_meta.get("content"):
+        author = author_meta["content"].strip()
+    else:
+        # Try common author class patterns
+        author_elem = soup.select_one(".author, .byline, [rel='author']")
+        if author_elem:
+            author = author_elem.get_text(strip=True)
+
+    if author:
+        metadata["author"] = author
+
+    # Extract published date
+    date = None
+    date_meta = soup.find("meta", property="article:published_time")
+    if date_meta and date_meta.get("content"):
+        date = date_meta["content"]
+    else:
+        date_elem = soup.select_one("time[datetime], .date, .published")
+        if date_elem:
+            date = date_elem.get("datetime") or date_elem.get_text(strip=True)
+
+    if date:
+        metadata["published_date"] = date
+
+    # Extract description
+    description = None
+    desc_meta = soup.find("meta", attrs={"name": "description"})
+    if desc_meta and desc_meta.get("content"):
+        description = desc_meta["content"].strip()
+
+    if description:
+        metadata["description"] = description
+
+    # Extract main content
+    content, extracted_title = extract_text_from_html(html)
+
+    # Use extracted title if we didn't find one
+    if not title and extracted_title:
+        metadata["title"] = extracted_title
+
+    return content, metadata

statement_extractor/document/loader.py

@@ -0,0 +1,303 @@
+"""
+URL loader for fetching and parsing web content.
+
+Orchestrates scraper and PDF parser plugins to load documents from URLs.
+"""
+
+import asyncio
+import logging
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from ..models.document import Document
+from ..pipeline.registry import PluginRegistry
+from ..plugins.base import (
+    BaseScraperPlugin,
+    BasePDFParserPlugin,
+    ContentType,
+    ScraperResult,
+)
+from .html_extractor import extract_text_from_html, extract_article_content
+
+logger = logging.getLogger(__name__)
+
+
+class URLLoaderConfig(BaseModel):
+    """Configuration for URL loading."""
+
+    timeout: float = Field(
+        default=30.0,
+        description="Request timeout in seconds"
+    )
+    use_ocr: bool = Field(
+        default=False,
+        description="Force OCR for PDF parsing"
+    )
+    max_pdf_pages: int = Field(
+        default=500,
+        description="Maximum pages to extract from PDFs"
+    )
+    scraper_plugin: Optional[str] = Field(
+        default=None,
+        description="Specific scraper plugin to use (None = auto-select)"
+    )
+    pdf_parser_plugin: Optional[str] = Field(
+        default=None,
+        description="Specific PDF parser plugin to use (None = auto-select)"
+    )
+    extract_metadata: bool = Field(
+        default=True,
+        description="Extract article metadata from HTML pages"
+    )
+
+
+class URLLoader:
+    """
+    Loads documents from URLs using scraper and PDF parser plugins.
+
+    Orchestrates the content acquisition process:
+    1. Fetch content using a scraper plugin
+    2. Detect content type (HTML vs PDF)
+    3. Parse content using appropriate parser
+    4. Create a Document object
+
+    Example:
+        >>> loader = URLLoader()
+        >>> document = await loader.load("https://example.com/article")
+        >>> print(document.title)
+
+        >>> # Synchronous usage
+        >>> document = loader.load_sync("https://example.com/report.pdf")
+    """
+
+    def __init__(self, config: Optional[URLLoaderConfig] = None):
+        """
+        Initialize the URL loader.
+
+        Args:
+            config: Loader configuration
+        """
+        self.config = config or URLLoaderConfig()
+        self._scraper: Optional[BaseScraperPlugin] = None
+        self._pdf_parser: Optional[BasePDFParserPlugin] = None
+
+    def _get_scraper(self) -> BaseScraperPlugin:
+        """Get the scraper plugin to use."""
+        if self._scraper is not None:
+            return self._scraper
+
+        scrapers = PluginRegistry.get_scrapers()
+        if not scrapers:
+            raise RuntimeError(
+                "No scraper plugins registered. "
+                "Ensure plugins are loaded via 'from statement_extractor import plugins'"
+            )
+
+        if self.config.scraper_plugin:
+            for scraper in scrapers:
+                if scraper.name == self.config.scraper_plugin:
+                    self._scraper = scraper
+                    return scraper
+            raise ValueError(f"Scraper plugin not found: {self.config.scraper_plugin}")
+
+        # Use first available (highest priority)
+        self._scraper = scrapers[0]
+        return self._scraper
+
+    def _get_pdf_parser(self) -> BasePDFParserPlugin:
+        """Get the PDF parser plugin to use."""
+        if self._pdf_parser is not None:
+            return self._pdf_parser
+
+        parsers = PluginRegistry.get_pdf_parsers()
+        if not parsers:
+            raise RuntimeError(
+                "No PDF parser plugins registered. "
+                "Ensure plugins are loaded via 'from statement_extractor import plugins'"
+            )
+
+        if self.config.pdf_parser_plugin:
+            for parser in parsers:
+                if parser.name == self.config.pdf_parser_plugin:
+                    self._pdf_parser = parser
+                    return parser
+            raise ValueError(f"PDF parser plugin not found: {self.config.pdf_parser_plugin}")
+
+        # Use first available (highest priority)
+        self._pdf_parser = parsers[0]
+        return self._pdf_parser
+
+    async def load(self, url: str) -> Document:
+        """
+        Load a URL and return a Document.
+
+        Args:
+            url: URL to load
+
+        Returns:
+            Document with extracted content
+
+        Raises:
+            ValueError: If URL cannot be fetched or parsed
+        """
+        logger.info(f"Loading URL: {url}")
+
+        # 1. Fetch content
+        scraper = self._get_scraper()
+        result = await scraper.fetch(url, self.config.timeout)
+
+        if not result.ok:
+            raise ValueError(f"Failed to fetch {url}: {result.error}")
+
+        logger.debug(f"Fetched {len(result.content)} bytes, type: {result.content_type}")
+
+        # 2. Process based on content type
+        if result.content_type == ContentType.PDF:
+            return self._process_pdf(result)
+        elif result.content_type == ContentType.HTML:
+            return self._process_html(result)
+        else:
+            # Try to guess based on content
+            if result.content[:5] == b"%PDF-":
+                return self._process_pdf(result)
+            # Default to HTML
+            return self._process_html(result)
+
+    def load_sync(self, url: str) -> Document:
+        """
+        Synchronous wrapper for load().
+
+        Args:
+            url: URL to load
+
+        Returns:
+            Document with extracted content
+        """
+        return asyncio.run(self.load(url))
+
+    def _process_pdf(self, result: ScraperResult) -> Document:
+        """
+        Convert PDF to Document with pages.
+
+        Args:
+            result: ScraperResult containing PDF bytes
+
+        Returns:
+            Document with PDF content
+        """
+        logger.info(f"Processing PDF from {result.final_url}")
+
+        parser = self._get_pdf_parser()
+        parse_result = parser.parse(
+            result.content,
+            max_pages=self.config.max_pdf_pages,
+            use_ocr=self.config.use_ocr,
+        )
+
+        if not parse_result.ok:
+            raise ValueError(f"Failed to parse PDF: {parse_result.error}")
+
+        logger.info(f"Extracted {len(parse_result.pages)} pages from PDF")
+
+        # Create Document from pages
+        kwargs = {
+            "pages": parse_result.pages,
+            "title": parse_result.metadata.get("title"),
+            "source_type": "pdf",
+            "url": result.final_url,
+        }
+        author = parse_result.metadata.get("author")
+        if author:
+            kwargs["authors"] = [author]
+
+        return Document.from_pages(**kwargs)
+
+    def _process_html(self, result: ScraperResult) -> Document:
+        """
+        Convert HTML to Document (single page).
+
+        Args:
+            result: ScraperResult containing HTML bytes
+
+        Returns:
+            Document with HTML content
+        """
+        logger.info(f"Processing HTML from {result.final_url}")
+
+        # Decode HTML
+        try:
+            html = result.content.decode("utf-8", errors="replace")
+        except Exception as e:
+            raise ValueError(f"Failed to decode HTML: {e}")
+
+        # Extract text and metadata
+        if self.config.extract_metadata:
+            text, metadata = extract_article_content(html)
+            title = metadata.get("title")
+            author = metadata.get("author")
+            # Log extracted metadata
+            logger.debug(f"Extracted metadata: {metadata}")
+        else:
+            text, title = extract_text_from_html(html)
+            author = None
+            metadata = {}
+
+        if not text or len(text.strip()) < 50:
+            raise ValueError("No meaningful content extracted from HTML")
+
+        logger.info(f"Extracted {len(text)} chars from HTML")
+        if title:
+            logger.info(f"  Title: {title}")
+        if author:
+            logger.info(f"  Author: {author}")
+        if metadata.get("published_date"):
+            logger.info(f"  Published: {metadata.get('published_date')}")
+
+        # Create Document using from_pages since from_text forces source_type="text"
+        kwargs = {
+            "pages": [text],
+            "title": title,
+            "source_type": "webpage",
+            "url": result.final_url,
+        }
+        if author:
+            kwargs["authors"] = [author]
+
+        return Document.from_pages(**kwargs)
+
+
+async def load_url(
+    url: str,
+    config: Optional[URLLoaderConfig] = None,
+) -> Document:
+    """
+    Convenience function to load a URL.
+
+    Args:
+        url: URL to load
+        config: Optional loader configuration
+
+    Returns:
+        Document with extracted content
+    """
+    loader = URLLoader(config)
+    return await loader.load(url)
+
+
+def load_url_sync(
+    url: str,
+    config: Optional[URLLoaderConfig] = None,
+) -> Document:
+    """
+    Convenience function to load a URL synchronously.
+
+    Args:
+        url: URL to load
+        config: Optional loader configuration
+
+    Returns:
+        Document with extracted content
+    """
+    loader = URLLoader(config)
+    return loader.load_sync(url)