osintengine 1.0.2__py3-none-any.whl

src/watson/core/models.py

@@ -0,0 +1,96 @@
+"""Core models for Watson — Pydantic data structures."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class FindingSeverity(str, Enum):
+    CRITICAL = "critical"
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+    INFO = "info"
+
+
+class FindingSource(str, Enum):
+    SATELLITE = "satellite"
+    GEOLOCATION = "geolocation"
+    IMAGE_VIDEO = "image_video"
+    SOCIAL_MEDIA = "social_media"
+    PEOPLE = "people"
+    WEBSITES = "websites"
+    CORPORATE = "corporate"
+    CONFLICT = "conflict"
+    OSINT = "osint"
+    BELLINGCAT = "bellingcat"
+    SOCMINT = "socmint"
+    CROSS_REF = "cross_ref"
+
+
+class Finding(BaseModel):
+    """A single finding from an investigation tool."""
+
+    id: str = Field(description="Unique finding ID")
+    source: FindingSource
+    tool: str = Field(description="Tool name that produced this finding")
+    title: str
+    description: str
+    evidence: list[str] = Field(default_factory=list, description="URLs or references")
+    severity: FindingSeverity = FindingSeverity.INFO
+    confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    timestamp: datetime = Field(default_factory=datetime.now)
+    metadata: dict = Field(default_factory=dict)
+
+
+class InvestigationTask(BaseModel):
+    """A decomposed investigation subtask dispatched to a tool."""
+
+    id: str
+    tool_category: FindingSource
+    tool_name: str = ""
+    query: str
+    context: str = ""
+    priority: int = 1
+
+
+class InvestigationRequest(BaseModel):
+    """Top-level investigation request from user."""
+
+    query: str
+    tools: list[FindingSource] | None = Field(default=None)
+    max_findings_per_tool: int = 10
+    cross_reference: bool = True
+
+
+class Report(BaseModel):
+    """Final investigation report."""
+
+    query: str
+    generated_at: datetime = Field(default_factory=datetime.now)
+    findings: list[Finding] = Field(default_factory=list)
+    cross_references: list[Finding] = Field(default_factory=list)
+    summary: str = ""
+    tool_stats: dict[str, int] = Field(default_factory=dict)
+
+    @property
+    def total_findings(self) -> int:
+        return len(self.findings) + len(self.cross_references)
+
+    @property
+    def by_severity(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for f in self.findings + self.cross_references:
+            counts[f.severity.value] = counts.get(f.severity.value, 0) + 1
+        return counts
+
+    @property
+    def by_source(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for f in self.findings:
+            counts[f.source.value] = counts.get(f.source.value, 0) + 1
+        return counts

src/watson/ethics.py

@@ -0,0 +1,205 @@
+"""Sidney Ledger — editorial compliance and entity tracking across investigations.
+
+Every entity that appears in a Watson investigation is recorded here.
+Cross-case patterns, harm assessments, and editorial gates live here.
+"""
+
+from __future__ import annotations
+import json
+import logging
+import os
+from pathlib import Path
+from datetime import datetime, timezone
+
+logger = logging.getLogger("watson.ethics")
+
+# ── Ledger ─────────────────────────────────────────────────────
+
+_LEDGER_PATH = Path(os.environ.get("WATSON_LEDGER_PATH",
+    os.path.expanduser("~/.watson/ledger.json")))
+
+
+class SidneyLedger:
+    """Cross-case entity ledger with editorial compliance."""
+    
+    def __init__(self, path: Path | None = None):
+        self.path = path or _LEDGER_PATH
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._data = self._load()
+    
+    def _load(self) -> dict:
+        if self.path.exists():
+            try:
+                return json.loads(self.path.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+        return {"entities": {}, "cases": {}, "version": "1.0"}
+    
+    def _save(self):
+        try:
+            self.path.write_text(json.dumps(self._data, indent=2, default=str))
+        except OSError as e:
+            logger.warning("ledger_save_failed: %s", e)
+    
+    def record_entity(self, name: str, entity_type: str, case_id: str, confidence: float,
+                      appears_in: list[str] | None = None):
+        """Record an entity found in an investigation."""
+        key = name.lower().strip()
+        if key not in self._data["entities"]:
+            self._data["entities"][key] = {
+                "canonical": name,
+                "type": entity_type,
+                "first_seen": datetime.now(timezone.utc).isoformat(),
+                "cases": [],
+                "total_appearances": 0,
+                "confidence_high": confidence,
+            }
+        
+        entry = self._data["entities"][key]
+        if case_id not in entry["cases"]:
+            entry["cases"].append(case_id)
+        entry["total_appearances"] = len(entry["cases"])
+        entry["confidence_high"] = max(entry["confidence_high"], confidence)
+        entry["last_seen"] = datetime.now(timezone.utc).isoformat()
+        
+        if appears_in:
+            existing = entry.get("appears_in", [])
+            for url in appears_in:
+                if url not in existing:
+                    existing.append(url)
+            entry["appears_in"] = existing[:50]
+        
+        self._save()
+    
+    def record_case(self, case_id: str, query: str, finding_count: int,
+                    confirmed_count: int, entities: list[str]):
+        """Record a completed investigation case."""
+        self._data["cases"][case_id] = {
+            "query": query,
+            "completed_at": datetime.now(timezone.utc).isoformat(),
+            "finding_count": finding_count,
+            "confirmed_count": confirmed_count,
+            "entities": entities,
+        }
+        self._save()
+    
+    def get_entity(self, name: str) -> dict | None:
+        """Look up an entity's cross-case history."""
+        return self._data["entities"].get(name.lower().strip())
+    
+    def get_case(self, case_id: str) -> dict | None:
+        """Look up a case."""
+        return self._data["cases"].get(case_id)
+    
+    def entity_appears_across_cases(self, name: str) -> list[str]:
+        """List all cases where this entity appears."""
+        entry = self.get_entity(name)
+        return entry["cases"] if entry else []
+    
+    def stats(self) -> dict:
+        return {
+            "total_entities": len(self._data["entities"]),
+            "total_cases": len(self._data["cases"]),
+            "cross_case_entities": sum(
+                1 for e in self._data["entities"].values()
+                if len(e.get("cases", [])) > 1
+            ),
+        }
+    
+    def get_stats(self) -> dict:
+        """Alias for stats() — used by health endpoint."""
+        return self.stats()
+
+
+# ── Editorial checks ───────────────────────────────────────────
+
+def apply_editorial_checks(findings: list, query: str) -> dict:
+    """Apply editorial compliance checks to findings. Returns assessment dict."""
+    
+    # Harm indicators — terms that suggest sensitive content
+    harm_indicators = {
+        "victim": "VULNERABLE ENTITIES DETECTED",
+        "minor": "MINOR DETECTED",
+        "child": "MINOR DETECTED",
+        "survivor": "VULNERABLE ENTITIES DETECTED",
+        "phone_number": "PII DETECTED",
+        "email": "PII DETECTED",
+        "address": "PII DETECTED",
+        "passport": "PII DETECTED",
+    }
+    
+    checks = {
+        "publication_assessment": "RECOMMENDED",
+        "harm_level": "LOW",
+        "public_interest": True,
+        "warnings": [],
+        "redactions_needed": 0,
+    }
+    
+    all_text = " ".join(
+        f"{getattr(f, 'title', '')} {getattr(f, 'description', '')}"
+        for f in findings
+    ).lower()
+    
+    for indicator, warning in harm_indicators.items():
+        if indicator in all_text:
+            checks["warnings"].append(warning)
+    
+    if checks["warnings"]:
+        checks["harm_level"] = "HIGH" if len(checks["warnings"]) > 2 else "MEDIUM"
+        checks["redactions_needed"] = len(checks["warnings"])
+    
+    return checks
+
+
+# ── Singleton ──────────────────────────────────────────────────
+
+_ledger: SidneyLedger | None = None
+
+
+def get_ledger() -> SidneyLedger:
+    global _ledger
+    if _ledger is None:
+        _ledger = SidneyLedger()
+    return _ledger
+
+
+# ── Editorial Framework (reporter integration) ──────────────────
+# These are lightweight stubs — the real editorial pipeline lives in
+# watson/reporter.py with the harm indicators in src/watson/ethics.py.
+
+BELLINGCAT_DATA_ETHICS_APPENDIX = """
+## Data Ethics & Methodology
+
+This investigation follows the Bellingcat Digital Research Ethics Framework:
+
+- **Open Source**: All data sourced from publicly available information
+- **Verification**: Cross-referenced across multiple independent sources
+- **Privacy**: Personal identifying information redacted unless clearly in the public interest
+- **Attribution**: Sources documented for reproducibility
+- **Correction**: Errors will be promptly corrected when identified
+- **Proportionality**: Only information relevant to the investigation is included
+
+Methodology: Multi-agent OSINT pipeline using web scraping, API queries,
+DNS/WHOIS lookups, certificate transparency logs, and dark web monitoring.
+"""
+
+
+def generate_compliance_header(assessment: dict) -> str:
+    """Generate a markdown compliance header from the assessment."""
+    warnings = assessment.get("warnings", [])
+    if not warnings:
+        return "✅ **Editorial Compliance**: No concerns detected.\n"
+    
+    lines = [f"⚠️  **Editorial Compliance** — {len(warnings)} concern(s):"]
+    for w in warnings:
+        lines.append(f"- {w}")
+    return "\n".join(lines) + "\n"
+
+
+class EditorialFramework:
+    """Editorial compliance framework for pre-publication assessment."""
+    
+    def assess(self, query: str, findings: list, narrative: str, ai_used: bool) -> dict:
+        """Assess a report for editorial compliance before publication."""
+        return apply_editorial_checks(findings, query)

src/watson/exports.py

@@ -0,0 +1,182 @@
+"""Export — generate reports in multiple formats.
+
+BellingcatReport class for structured OSINT report generation.
+Delegates to watson.reporter for core report logic.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+class BellingcatReport:
+    """Bellingcat-style structured OSINT report with multi-format export.
+
+    Supports: JSON, STIX, MISP, PDF, Markdown.
+    """
+
+    def __init__(
+        self,
+        query: str = "",
+        investigation_id: str = "",
+        target_type: str = "unknown",
+        target_value: str = "",
+    ):
+        self.query = query
+        self.investigation_id = investigation_id
+        self.target_type = target_type
+        self.target_value = target_value
+        self.findings: List[Dict[str, Any]] = []
+        self.cross_references: List[Dict[str, Any]] = []
+        self.hops: int = 0
+
+    def add_findings(self, findings: List[Dict[str, Any]]):
+        self.findings.extend(findings)
+
+    def add_cross_references(self, refs: List[Dict[str, Any]]):
+        self.cross_references.extend(refs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return full report as dict."""
+        return {
+            "case_id": self.investigation_id,
+            "query": self.query,
+            "target_type": self.target_type,
+            "target_value": self.target_value,
+            "hops": self.hops,
+            "total_findings": len(self.findings),
+            "confirmed": sum(1 for f in self.findings if f.get("tier") == "CONFIRMED"),
+            "findings": self.findings,
+            "cross_references": self.cross_references,
+            "verifiability": self._verifiability(),
+        }
+
+    def to_json(self, path: Path | str) -> Path:
+        """Export as JSON."""
+        path = Path(path)
+        path.write_text(json.dumps(self.to_dict(), indent=2, default=str))
+        return path
+
+    def to_stix(self, path: Path | str) -> Path:
+        """Export as STIX 2.1 bundle."""
+        import uuid
+        from datetime import datetime, timezone
+
+        now = datetime.now(timezone.utc).isoformat()
+        bundle = {
+            "type": "bundle",
+            "id": f"bundle--{uuid.uuid4()}",
+            "spec_version": "2.1",
+            "objects": [
+                {
+                    "type": "report",
+                    "id": f"report--{uuid.uuid4()}",
+                    "created": now,
+                    "modified": now,
+                    "name": f"Watson Investigation: {self.query[:80]}",
+                    "description": f"OSINT investigation of {self.target_value}",
+                    "report_types": ["investigation"],
+                    "object_refs": [],
+                }
+            ],
+        }
+        path = Path(path)
+        path.write_text(json.dumps(bundle, indent=2))
+        return path
+
+    def to_misp(self, path: Path | str) -> Path:
+        """Export as MISP event JSON."""
+        import uuid
+
+        event = {
+            "Event": {
+                "uuid": str(uuid.uuid4()),
+                "info": f"Watson Investigation: {self.query[:80]}",
+                "analysis": "2",  # Completed
+                "threat_level_id": "3",  # Low
+                "published": False,
+                "Attribute": [
+                    {
+                        "type": "text",
+                        "category": "External analysis",
+                        "value": f["title"],
+                        "comment": f["description"],
+                    }
+                    for f in self.findings[:50]
+                ],
+            }
+        }
+        path = Path(path)
+        path.write_text(json.dumps(event, indent=2))
+        return path
+
+    def to_pdf(self, path: Path | str) -> Path:
+        """Export as PDF (saves as JSON with .pdf extension if fpdf unavailable)."""
+        path = Path(path)
+        try:
+            from fpdf import FPDF
+            pdf = FPDF()
+            pdf.add_page()
+            pdf.set_font("Helvetica", size=12)
+            pdf.cell(200, 10, text=f"Watson OSINT Report: {self.query[:80]}", align="C")
+            pdf.ln(10)
+            for f in self.findings:
+                pdf.set_font("Helvetica", "B", size=10)
+                pdf.cell(200, 8, text=f["title"][:100])
+                pdf.ln(8)
+                pdf.set_font("Helvetica", size=9)
+                pdf.multi_cell(0, 6, text=f.get("description", "")[:300])
+                pdf.ln(4)
+            pdf.output(str(path))
+        except ImportError:
+            # Fallback: save as markdown with .pdf extension
+            path.write_text(self.to_markdown())
+        return path
+
+    def to_markdown(self) -> str:
+        """Export as Markdown."""
+        lines = [
+            f"# Watson OSINT Investigation Report",
+            f"",
+            f"**Case ID:** {self.investigation_id}",
+            f"**Query:** {self.query}",
+            f"**Target Type:** {self.target_type}",
+            f"**Target:** {self.target_value}",
+            f"**Hops:** {self.hops}",
+            f"**Findings:** {len(self.findings)}",
+            f"**Verifiability:** {self._verifiability():.0%}",
+            f"",
+            f"## Findings",
+            f"",
+        ]
+        for i, f in enumerate(self.findings, 1):
+            tier = f.get("tier", "UNSUBSTANTIATED")
+            lines.append(f"### Finding {i}: {f.get('title', 'Untitled')}")
+            lines.append(f"")
+            lines.append(f"- **Tier:** {tier}")
+            lines.append(f"- **Confidence:** {f.get('confidence', 0):.1%}")
+            lines.append(f"- **Source:** {f.get('source_type', 'unknown')}")
+            if f.get("source_url"):
+                lines.append(f"- **URL:** {f.get('source_url')}")
+            lines.append(f"")
+            lines.append(f.get("description", ""))
+            lines.append(f"")
+
+        if self.cross_references:
+            lines.append(f"## Cross-References")
+            lines.append(f"")
+            for cr in self.cross_references:
+                lines.append(f"- {cr}")
+
+        return "\n".join(lines)
+
+    def _verifiability(self) -> float:
+        """Calculate the verifiability score (0-1)."""
+        if not self.findings:
+            return 0.0
+        sources = sum(1 for f in self.findings if f.get("source_url"))
+        primary = sum(1 for f in self.findings if f.get("source_type") == "PRIMARY")
+        score = (sources / len(self.findings) * 0.5 + primary / max(len(self.findings), 1) * 0.5)
+        return min(score, 1.0)

src/watson/graph/__init__.py

@@ -0,0 +1,69 @@
+"""Watson Entity Graph — typed entities, relationships, and transform engine.
+
+This module provides Maltego-style graph-based OSINT discovery:
+  - Typed entities (Domain, IPAddress, Email, Person, Website, etc.)
+  - Typed relationships (RESOLVES_TO, HAS_SUBDOMAIN, LOCATED_IN, etc.)
+  - EntityGraph: a networkx-backed graph with type-safe operations
+  - TransformEngine: recursive entity→entity transform chaining
+
+Integration: the graph engine runs as an enrichment layer AFTER the surface
+phase of the existing pipeline. It does NOT modify the pipeline — it adds
+new findings derived from the graph transforms, while all existing phases
+continue unchanged.
+
+Usage:
+    from watson.graph import EntityGraph, TransformEngine
+    g = EntityGraph()
+    g.add_entity(Domain("automattic.com"))
+    engine = TransformEngine(g)
+    await engine.run_transforms(max_depth=3)
+    findings = g.to_findings()
+"""
+
+from .entities import (
+    Entity,
+    Domain,
+    IPAddress,
+    Email,
+    Person,
+    Organization,
+    Website,
+    Location,
+    Document,
+)
+from .relationships import (
+    Relationship,
+    RESOLVES_TO,
+    HAS_SUBDOMAIN,
+    LOCATED_IN,
+    HAS_EMAIL,
+    MENTIONS,
+    CONTAINS,
+    OWNS,
+    RELATED_TO,
+)
+from .graph import EntityGraph
+from .transforms import TransformEngine
+
+__all__ = [
+    "Entity",
+    "Domain",
+    "IPAddress",
+    "Email",
+    "Person",
+    "Organization",
+    "Website",
+    "Location",
+    "Document",
+    "Relationship",
+    "RESOLVES_TO",
+    "HAS_SUBDOMAIN",
+    "LOCATED_IN",
+    "HAS_EMAIL",
+    "MENTIONS",
+    "CONTAINS",
+    "OWNS",
+    "RELATED_TO",
+    "EntityGraph",
+    "TransformEngine",
+]