contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/extraction/extractors/website_extractor.py

@@ -0,0 +1,246 @@
+"""
+Website/URL Extractor
+
+Extracts content from websites using trafilatura.
+"""
+
+from pathlib import Path
+from typing import Dict, Any
+import logging
+import json
+
+from ..registry import BaseExtractor, ExtractionResult
+
+logger = logging.getLogger(__name__)
+
+
+class WebsiteExtractor(BaseExtractor):
+    """
+    Website extractor using trafilatura for clean text extraction.
+    """
+    
+    def __init__(self, config):
+        """Initialize website extractor."""
+        super().__init__(config)
+        self.extractor_version = "website_v1"
+    
+    def supports_format(self, file_path: str) -> bool:
+        """Check if input is a URL or website."""
+        path_str = str(file_path).lower().strip()
+        return (
+            path_str.startswith("http://") or
+            path_str.startswith("https://") or
+            path_str.startswith("www.")
+        )
+    
+    def extract(self, file_path: str) -> ExtractionResult:
+        """
+        Extract content from website/URL.
+        
+        Args:
+            file_path: URL or website address
+            
+        Returns:
+            ExtractionResult
+        """
+        result = ExtractionResult(extractor_version=self.extractor_version)
+        
+        # #region agent log
+        log_path = r"c:\qgraph\qgraph-app\.cursor\debug.log"
+        try:
+            with open(log_path, 'a', encoding='utf-8') as f:
+                f.write(json.dumps({"sessionId":"debug-session","runId":"extraction","hypothesisId":"C","location":"website_extractor.py:40","message":"Starting website extraction","data":{"url":file_path},"timestamp":int(__import__('time').time()*1000)})+"\n")
+        except: pass
+        # #endregion
+        
+        try:
+            import trafilatura
+            import requests
+            from urllib.parse import urlparse
+            
+            # Normalize URL
+            url = str(file_path).strip()
+            if not url.startswith(("http://", "https://")):
+                if url.startswith("www."):
+                    url = "https://" + url
+                else:
+                    url = "https://" + url
+            
+            # Fetch HTML content
+            try:
+                headers = {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                }
+                response = requests.get(url, headers=headers, timeout=30)
+                response.raise_for_status()
+                html_content = response.text
+            except Exception as e:
+                logger.error(f"Failed to fetch URL {url}: {e}")
+                result.errors.append(f"Failed to fetch URL: {e}")
+                return result
+            
+            # Extract main content using trafilatura
+            # Extract text content (default format)
+            extracted = trafilatura.extract(html_content, output_format='txt')
+            
+            # Extract XML format for structured output (optional)
+            extracted_xml = None
+            try:
+                extracted_xml = trafilatura.extract(html_content, output_format='xml')
+            except:
+                pass
+            
+            if not extracted:
+                # Fallback: use BeautifulSoup to extract text
+                try:
+                    from bs4 import BeautifulSoup
+                    soup = BeautifulSoup(html_content, 'html.parser')
+                    # Remove script and style elements
+                    for script in soup(["script", "style"]):
+                        script.decompose()
+                    extracted = soup.get_text(separator='\n', strip=True)
+                except:
+                    extracted = ""
+            
+            # Extract metadata
+            metadata_dict = None
+            try:
+                metadata_dict = trafilatura.extract_metadata(html_content)
+            except:
+                pass
+            
+            # Use extracted text as text_content
+            text_content = extracted or ""
+            
+            # Extract tables from HTML
+            try:
+                from bs4 import BeautifulSoup
+                soup = BeautifulSoup(html_content, 'html.parser')
+                tables = soup.find_all('table')
+                
+                for table_idx, table in enumerate(tables, 1):
+                    table_data = []
+                    rows = table.find_all('tr')
+                    for row in rows:
+                        cells = row.find_all(['td', 'th'])
+                        row_data = [cell.get_text(strip=True) for cell in cells]
+                        if row_data:
+                            table_data.append(row_data)
+                    
+                    if table_data:
+                        result.tables.append({
+                            "id": f"tbl_{table_idx}",
+                            "data": table_data,
+                            "headers": table_data[0] if table_data else [],
+                            "rows": table_data[1:] if len(table_data) > 1 else [],
+                        })
+            except ImportError:
+                logger.warning("BeautifulSoup not available for table extraction")
+            except Exception as e:
+                logger.warning(f"Table extraction failed: {e}")
+            
+            # Extract images
+            try:
+                from bs4 import BeautifulSoup
+                soup = BeautifulSoup(html_content, 'html.parser')
+                images = soup.find_all('img')
+                
+                for img_idx, img in enumerate(images, 1):
+                    src = img.get('src', '')
+                    alt = img.get('alt', '')
+                    
+                    # Resolve relative URLs
+                    if src and not src.startswith(('http://', 'https://')):
+                        parsed_url = urlparse(url)
+                        base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+                        if src.startswith('/'):
+                            src = base_url + src
+                        else:
+                            src = base_url + '/' + src
+                    
+                    result.images.append({
+                        "id": f"img_{img_idx}",
+                        "path": src,
+                        "description": alt,
+                        "url": src,
+                    })
+            except ImportError:
+                logger.warning("BeautifulSoup not available for image extraction")
+            except Exception as e:
+                logger.warning(f"Image extraction failed: {e}")
+            
+            # Build metadata
+            result.metadata = {
+                "title": metadata_dict.title if metadata_dict else urlparse(url).netloc,
+                "author": metadata_dict.author if metadata_dict else "",
+                "url": url,
+                "domain": urlparse(url).netloc,
+                "page_count": 1,
+            }
+            
+            # Store as page
+            result.pages.append({
+                "page_no": 1,
+                "text": text_content,
+                "content": text_content,
+            })
+            
+            # Store web content
+            result.web.append({
+                "url": url,
+                "html": html_content,
+                "clean_text": text_content,
+                "extracted_xml": extracted_xml if extracted_xml else "",
+            })
+            
+            # #region agent log
+            try:
+                with open(log_path, 'a', encoding='utf-8') as f:
+                    f.write(json.dumps({"sessionId":"debug-session","runId":"extraction","hypothesisId":"C","location":"website_extractor.py:155","message":"Website extraction complete","data":{"text_length":len(text_content),"tables_extracted":len(result.tables),"images_extracted":len(result.images)},"timestamp":int(__import__('time').time()*1000)})+"\n")
+            except: pass
+            # #endregion
+            
+        except ImportError:
+            error_msg = "trafilatura or requests not available. Install with: pip install trafilatura requests beautifulsoup4"
+            logger.error(error_msg)
+            result.errors.append(error_msg)
+        except Exception as e:
+            logger.error(f"Website extraction failed: {e}")
+            result.errors.append(str(e))
+            # #region agent log
+            try:
+                with open(log_path, 'a', encoding='utf-8') as f:
+                    f.write(json.dumps({"sessionId":"debug-session","runId":"extraction","hypothesisId":"C","location":"website_extractor.py:167","message":"Website extraction error","data":{"error":str(e)},"timestamp":int(__import__('time').time()*1000)})+"\n")
+            except: pass
+            # #endregion
+        
+        return result
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

contextsynapse/extraction/fact_extractor.py

@@ -0,0 +1,236 @@
+"""
+Fact Extractor
+===============
+Extracts verifiable facts from text using LLM — claims, metrics,
+dates, decisions — and creates graph nodes linked to source chunks.
+
+Facts are first-class citizens in the knowledge graph:
+  - Each fact becomes a node (Claim, Metric, Date, Decision)
+  - Linked to source Document/Chunk via EXTRACTED_FROM edge
+  - Confidence scores for reliability filtering
+  - Source text preserved for verification
+
+Usage:
+    from contextsynapse.extraction.fact_extractor import FactExtractor
+
+    extractor = FactExtractor(db)
+    facts = extractor.extract_facts("Revenue was $4.2M in Q1, up 15%.")
+    # → Metric(name="Revenue", value="4.2M", unit="USD", period="Q1")
+    # → Claim(statement="Revenue up 15%", confidence=0.9)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+FACT_EXTRACTION_PROMPT = """Extract all verifiable facts from the following text.
+
+For each fact, identify:
+- type: one of Claim, Metric, Date, Decision, Requirement, Definition
+- statement: the fact in one clear sentence
+- confidence: how certain this fact is (0.0 to 1.0)
+- source_text: the exact quote from the text that supports this fact
+- properties: type-specific fields:
+  - Metric: {name, value, unit, period, comparison}
+  - Date: {event, date, description}
+  - Decision: {title, rationale, decided_by, status}
+  - Requirement: {title, priority, status}
+  - Definition: {term, definition}
+  - Claim: {subject, predicate}
+
+Return JSON:
+{
+  "facts": [
+    {
+      "type": "Metric",
+      "statement": "Q1 revenue was $4.2M",
+      "confidence": 0.95,
+      "source_text": "Revenue reached $4.2 million in the first quarter",
+      "properties": {"name": "Revenue", "value": "4.2M", "unit": "USD", "period": "Q1"}
+    }
+  ]
+}
+
+Text to extract facts from:
+"""
+
+FACT_SYSTEM_PROMPT = (
+    "You are a fact extraction engine. Extract precise, verifiable facts from text. "
+    "Be thorough — capture metrics, dates, decisions, claims, requirements, and definitions. "
+    "Assign confidence scores honestly. Always return valid JSON."
+)
+
+
+@dataclass
+class Fact:
+    """A single extracted fact."""
+    fact_type: str
+    statement: str
+    confidence: float
+    source_text: str
+    properties: Dict[str, Any] = field(default_factory=dict)
+    node_id: Optional[str] = None  # set after graph creation
+
+
+@dataclass
+class FactExtractionResult:
+    """Result of fact extraction."""
+    facts: List[Fact] = field(default_factory=list)
+    node_ids: List[str] = field(default_factory=list)
+    edge_ids: List[str] = field(default_factory=list)
+
+
+class FactExtractor:
+    """Extract verifiable facts from text and create graph nodes."""
+
+    def __init__(
+        self,
+        db=None,
+        provider: Optional[str] = None,
+        model: Optional[str] = None,
+    ):
+        self.db = db
+        self.provider = provider
+        self.model = model
+
+    def extract_facts(
+        self,
+        text: str,
+        source_node_id: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        fact_types: Optional[List[str]] = None,
+    ) -> FactExtractionResult:
+        """Extract facts from text and optionally create graph nodes.
+
+        Args:
+            text: Text to extract facts from.
+            source_node_id: Optional node ID to link facts via EXTRACTED_FROM.
+            agent_id: Agent performing extraction.
+            fact_types: Limit to these fact types (None = all).
+
+        Returns:
+            FactExtractionResult with facts and created node/edge IDs.
+        """
+        from ..llm import get_llm_client
+
+        llm = get_llm_client(provider=self.provider, model=self.model)
+
+        prompt = FACT_EXTRACTION_PROMPT + text
+        if fact_types:
+            prompt = prompt.replace(
+                "one of Claim, Metric, Date, Decision, Requirement, Definition",
+                f"one of {', '.join(fact_types)}",
+            )
+
+        raw = llm.generate_json(prompt=prompt, system=FACT_SYSTEM_PROMPT)
+        raw_facts = raw.get("facts", [])
+
+        result = FactExtractionResult()
+
+        for rf in raw_facts:
+            fact = Fact(
+                fact_type=rf.get("type", "Claim"),
+                statement=rf.get("statement", ""),
+                confidence=float(rf.get("confidence", 0.5)),
+                source_text=rf.get("source_text", ""),
+                properties=rf.get("properties", {}),
+            )
+
+            # Filter by requested types
+            if fact_types and fact.fact_type not in fact_types:
+                continue
+
+            result.facts.append(fact)
+
+            # Create graph node if db is available
+            if self.db:
+                node_id = self._create_fact_node(fact, agent_id)
+                fact.node_id = node_id
+                result.node_ids.append(node_id)
+
+                # Link to source node
+                if source_node_id and node_id:
+                    edge_id = self._create_source_edge(node_id, source_node_id, agent_id)
+                    if edge_id:
+                        result.edge_ids.append(edge_id)
+
+        logger.info("Extracted %d facts from %d chars", len(result.facts), len(text))
+        return result
+
+    def extract_facts_from_chunks(
+        self,
+        chunks: List[Dict[str, str]],
+        agent_id: Optional[str] = None,
+    ) -> FactExtractionResult:
+        """Extract facts from multiple chunks.
+
+        Args:
+            chunks: List of {"text": "...", "node_id": "..."} dicts.
+            agent_id: Agent performing extraction.
+        """
+        combined = FactExtractionResult()
+
+        for chunk in chunks:
+            text = chunk.get("text", "")
+            node_id = chunk.get("node_id")
+
+            if len(text.strip()) < 30:
+                continue
+
+            try:
+                result = self.extract_facts(
+                    text, source_node_id=node_id, agent_id=agent_id,
+                )
+                combined.facts.extend(result.facts)
+                combined.node_ids.extend(result.node_ids)
+                combined.edge_ids.extend(result.edge_ids)
+            except Exception as e:
+                logger.warning("Fact extraction failed for chunk: %s", e)
+
+        return combined
+
+    def _create_fact_node(self, fact: Fact, agent_id: Optional[str] = None) -> str:
+        """Create a graph node for a fact."""
+        from ..core.graph_structures import GraphNode
+
+        node_id = str(uuid.uuid4())
+        props = {
+            "statement": fact.statement,
+            "confidence": fact.confidence,
+            "source_text": fact.source_text,
+            "_source": "fact_extraction",
+            **fact.properties,
+        }
+        if agent_id:
+            props["_agent_id"] = agent_id
+
+        node = GraphNode(id=node_id, label=fact.fact_type, properties=props)
+        self.db.add_node(node, write_through=True)
+        return node_id
+
+    def _create_source_edge(
+        self, fact_node_id: str, source_node_id: str, agent_id: Optional[str] = None,
+    ) -> Optional[str]:
+        """Create EXTRACTED_FROM edge linking fact to source."""
+        from ..core.graph_structures import GraphEdge
+
+        edge_id = str(uuid.uuid4())
+        props = {"_source": "fact_extraction"}
+        if agent_id:
+            props["_agent_id"] = agent_id
+
+        edge = GraphEdge(
+            id=edge_id, source=fact_node_id, target=source_node_id,
+            label="EXTRACTED_FROM", properties=props,
+        )
+        try:
+            self.db.add_edge(edge)
+            return edge_id
+        except Exception:
+            return None