osintengine 1.0.2__tar.gz → 1.0.4__tar.gz

{osintengine-1.0.2 → osintengine-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: osintengine
-Version: 1.0.2
+Version: 1.0.4
 Summary: The OSINT investigation engine — 7-phase pipeline, persistent knowledge graph, community intelligence
 Author-email: Lorenzo Baron <baron.lorenzo99@gmail.com>
 License-Expression: Apache-2.0
@@ -65,9 +65,9 @@ Dynamic: license-file
 </p>
 
 <p align="center">
-  <a href="https://pypi.org/project/osintengine/"><img src="https://img.shields.io/badge/pypi-v1.0.2-blue" alt="PyPI"></a>
+  <a href="https://pypi.org/project/osintengine/"><img src="https://img.shields.io/badge/pypi-v1.0.3-blue" alt="PyPI"></a>
   <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python"></a>
-  <a href="LICENSE"><img src="https://img.shields.io/badge/license-AGPLv3-green" alt="License"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-green" alt="License"></a>
   <a href="https://github.com/Lorenzobaron99/watson-osint"><img src="https://img.shields.io/badge/status-active-brightgreen" alt="Status"></a>
 </p>
 
@@ -85,7 +85,7 @@ One target. One command. A dossier that would take an analyst hours.
 
 | You type... | Most tools return... | Watson returns... |
 |---|---|---|
-| "Paolo Trecate" | Wikipedia link, maybe a news article | Employment history, board positions, corporate registrations, domain ownerships, sanctions screening, adverse media, timeline of professional milestones, entity relationship graph |
+| "John Chen" | Wikipedia link, maybe a news article | Employment history, board positions, corporate registrations, domain ownerships, sanctions screening, adverse media, timeline of professional milestones, entity relationship graph |
 | "example.com" | WHOIS record | Domain history, SSL certificates, subdomains, DNS records, IP geolocation, hosting provider, Wayback snapshots, linked social accounts, VirusTotal reputation |
 | "crypto-wallet-0x..." | Nothing useful | Transaction graph, exchange links, associated addresses, risk scoring, dark web mentions |
 

{osintengine-1.0.2 → osintengine-1.0.4}/README.md

@@ -9,9 +9,9 @@
 </p>
 
 <p align="center">
-  <a href="https://pypi.org/project/osintengine/"><img src="https://img.shields.io/badge/pypi-v1.0.2-blue" alt="PyPI"></a>
+  <a href="https://pypi.org/project/osintengine/"><img src="https://img.shields.io/badge/pypi-v1.0.3-blue" alt="PyPI"></a>
   <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python"></a>
-  <a href="LICENSE"><img src="https://img.shields.io/badge/license-AGPLv3-green" alt="License"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-green" alt="License"></a>
   <a href="https://github.com/Lorenzobaron99/watson-osint"><img src="https://img.shields.io/badge/status-active-brightgreen" alt="Status"></a>
 </p>
 
@@ -29,7 +29,7 @@ One target. One command. A dossier that would take an analyst hours.
 
 | You type... | Most tools return... | Watson returns... |
 |---|---|---|
-| "Paolo Trecate" | Wikipedia link, maybe a news article | Employment history, board positions, corporate registrations, domain ownerships, sanctions screening, adverse media, timeline of professional milestones, entity relationship graph |
+| "John Chen" | Wikipedia link, maybe a news article | Employment history, board positions, corporate registrations, domain ownerships, sanctions screening, adverse media, timeline of professional milestones, entity relationship graph |
 | "example.com" | WHOIS record | Domain history, SSL certificates, subdomains, DNS records, IP geolocation, hosting provider, Wayback snapshots, linked social accounts, VirusTotal reputation |
 | "crypto-wallet-0x..." | Nothing useful | Transaction graph, exchange links, associated addresses, risk scoring, dark web mentions |
 

{osintengine-1.0.2 → osintengine-1.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "osintengine"
-version = "1.0.2"
+version = "1.0.4"
 description = "The OSINT investigation engine — 7-phase pipeline, persistent knowledge graph, community intelligence"
 readme = "README.md"
 license = "Apache-2.0"

{osintengine-1.0.2 → osintengine-1.0.4}/src/watson/__init__.py

@@ -7,4 +7,4 @@ must be the truth." — Sherlock Holmes
 Autonomous open-source intelligence with 338 Bellingcat tools.
 Knowledge graph, lead tracking, recursive investigation.
 """
-__version__ = "1.0.2"
+__version__ = "1.0.4"

{osintengine-1.0.2 → osintengine-1.0.4}/src/watson/orchestration/engine.py

@@ -1927,7 +1927,7 @@ Examples:
                             # FORCE a targeted person search excluding the non-person Wikipedia hit
                             # Universal fix: when Wikipedia returns a location/org/thing instead of a person,
                             # search with -site:wikipedia.org + person keywords. Works for:
-                            # - Paolo Trecate (comune) → "Paolo Trecate" analyst defense
+                            # - Marco Rossi (comune) → "Marco Rossi" analyst defense
                             # - Austin Texas (city) → "Austin Texas" linkedin
                             # - Paris Hilton (city) → "Paris Hilton" celebrity (though common enough to find anyway)
                             try:
@@ -2808,7 +2808,7 @@ Examples:
                 # employment, board positions) — essential context even for criminal
                 # investigations. Criminal queries add sanctions, court records, and
                 # wanted notices. Without professional queries, deep investigations
-                # miss entire professional profiles (e.g. "Paolo Trecate" → only found
+                # miss entire professional profiles (e.g. "Marco Rossi" → only found
                 # jail lookup sites, never discovered eurodefense.tech).
                 search_label = "searching professional history + court records, legal databases"
                 search_mode = "both"
@@ -3616,7 +3616,9 @@ Examples:
         except Exception as e:
             logger.warning("resolution_failed: %s", e)
 
-        # Synthesis
+        # Synthesis — filter noise before feeding to LLM
+        findings = self._filter_quality(findings, query=query)
+        
         try:
             from .synthesis import synthesize_brief
             from .llm_config import call_llm
@@ -3802,14 +3804,14 @@ Examples:
         primary = sum(1 for f in findings if f.source_tier == "PRIMARY")
         has_url = sum(1 for f in findings if f.source_url)
 
-        # Verifiability score — weighted: URL sources + high-confidence tiers
-        # Most OSINT findings are SECONDARY (news, Wikipedia) — PRIMARY tier
-        # (court docs, sanctions lists) is rare. Weight PROBABLE heavily.
+        # Verifiability score — weighted: confirmed findings + primary sources dominate.
+        # CONFIRMED findings are the gold standard; PRIMARY sources (court docs, sanctions)
+        # carry the most weight. PROBABLE adds marginal confidence.
         report.verifiability_score = (
-            0.25 * (primary / max(total, 1)) +
+            0.35 * (confirmed / max(total, 1)) +
+            0.30 * (primary / max(total, 1)) +
             0.25 * (has_url / max(total, 1)) +
-            0.15 * (confirmed / max(total, 1)) +
-            0.35 * (probable / max(total, 1))
+            0.10 * (probable / max(total, 1))
         )
 
         lines = [
@@ -4830,7 +4832,7 @@ and confidence assessments. Follow the OUTPUT FORMAT specified above."""
         """Compute token-overlap relevance between target query and finding text.
 
         Returns 0.0 – 1.0. A score of 0 means zero word overlap — the finding
-        is almost certainly unrelated (e.g., "Idaho murders" for "BÓDI Ildikó").
+        is almost certainly unrelated (e.g., "Idaho murders" for "KOVÁCS Anna").
         """
         import unicodedata
         if not target or not text:
@@ -5004,7 +5006,7 @@ and confidence assessments. Follow the OUTPUT FORMAT specified above."""
             "no family members identified",
             "no specific co-conspirator names",
             "no specific fine", "no specific asset",
-            # ── Leaked through BÓDI Ildikó investigation ──
+            # ── Leaked through KOVÁCS investigation ──
             "no specific legal citations",
             "no court identifiers",
             "no child details",
@@ -5111,6 +5113,24 @@ and confidence assessments. Follow the OUTPUT FORMAT specified above."""
                 "i will run", "i will simulate", "i must ask",
                 "you have not", "if you can", "instruct me",
                 "once you provide", "example of what",
+                # ── Leaked through Mohammadzadeh investigation ──
+                "let's first extract", "let's first",
+                "i will open", "i'll open", "i can use python",
+                "i'm not actually able", "i recall the",
+                "for each fact", "i'll also", "i'll keep",
+                "i'll treat", "i'll use", "i can use",
+                "the youtube video is likely",
+                "however, the user also",
+                "i will incorporate", "i can incorporate",
+                "i could open", "i could use",
+                "i don't have the full", "i can view",
+                "i could mention", "i'll rely on",
+                "but i can't see", "i can't see the full",
+                "the search result snippet says",
+                "for iran:", "i can mention that",
+                "as an ai in this simulation",
+                "i need to fetch", "i might need",
+                "i will treat", "i can mention",
             ])
 
             if is_meta and not (has_proper_names or has_dates or has_source_ref):
@@ -5121,6 +5141,22 @@ and confidence assessments. Follow the OUTPUT FORMAT specified above."""
                 f.source_tier = "UNVERIFIED"
                 f.confidence = min(f.confidence, 0.25)
 
+            # ── Drop: DNS/IP resolution noise when target is a person ──
+            # Graph enrichment resolves every domain in findings to IPs and geolocates them.
+            # For person investigations, this produces 14+ noise findings (Cloudflare → Toronto).
+            # Only keep DNS/IP enrichment when the investigation target IS a domain or IP.
+            if getattr(f, 'phase', '') == 'graph_enrich':
+                tf = (f.title or '').lower()
+                df = (f.description or '').lower()
+                # DNS resolution or IP geolocation that's about infrastructure, not the target
+                dns_patterns = [
+                    'dns a record', 'resolves_to', '→ resolves_to',
+                    '→ located_in', 'located_in', 'ip geolocation',
+                    'cloudflare, inc.', '→ toronto', '→ san francisco',
+                ]
+                if any(p in tf or p in df for p in dns_patterns):
+                    continue
+
             kept.append(f)
 
         # ── Relevance check: universal pre-synthesis pipeline (any target type, any language) ──

{osintengine-1.0.2 → osintengine-1.0.4}/src/watson/orchestration/synthesis.py

@@ -560,37 +560,45 @@ def _fallback_brief(query: str, findings: list) -> dict:
         "Financial": ["revenue", "profit", "loss", "billion", "million", "funding", "valuation",
                       "stock", "share", "investor", "ipo", "acquisition"],
         "Crime / Investigation": ["criminal", "arrest", "prison", "jail", "fraud", "money laundering",
-                                   "investigation", "probe", "allegation", "misconduct"],
-        "Infrastructure / Technical": ["dns", "ip address", "nameserver", "hosting", "cdn",
-                                        "subdomain", "resolves_to", "mx record"],
+                                   "investigation", "probe", "allegation", "misconduct", "wanted", "fugitive",
+                                   "fbi", "hacker", "cyber"],
     }
     themes = []
     all_text = " ".join(titles + [getattr(f, "description", "") or "" for f in findings]).lower()
     for theme, keywords in theme_keywords.items():
         matches = [kw for kw in keywords if kw in all_text]
         if matches:
-            # Extract a relevant finding snippet
+            # Find the best finding for this theme — prefer ones with real descriptions
+            best_summary = f"Evidence of {matches[0]} found across multiple sources."
+            best_source = ""
             for f in findings:
-                desc = (getattr(f, "description", "") or "").lower()
+                desc = (getattr(f, "description", "") or "")
                 title_l = (getattr(f, "title", "") or "").lower()
                 for kw in matches:
-                    if kw in title_l or kw in desc:
-                        themes.append({
-                            "theme": theme,
-                            "severity": "HIGH" if theme.startswith("Crime") else "MEDIUM",
-                            "summary": getattr(f, "title", "")[:200] or f"Evidence of {kw} found",
-                            "source_titles": [getattr(f, "title", "")[:100]],
-                        })
+                    if kw in title_l or kw in desc.lower():
+                        # Use the description as summary (actual intelligence, not just title)
+                        if desc and len(desc) > 20:
+                            best_summary = desc[:300]
+                            best_source = getattr(f, "source_url", "") or ""
                         break
-                if any(t["theme"] == theme for t in themes):
+                if best_source:
                     break
-            else:
-                themes.append({
-                    "theme": theme,
-                    "severity": "MEDIUM",
-                    "summary": f"Keywords detected: {', '.join(matches[:4])}",
-                    "source_titles": [],
-                })
+            themes.append({
+                "theme": theme,
+                "severity": "HIGH" if theme.startswith("Crime") else "MEDIUM",
+                "summary": best_summary,
+                "source_titles": [best_source] if best_source else [],
+            })
+    
+    # Deduplicate themes — same summary shouldn't appear for multiple themes
+    seen_summaries = set()
+    deduped_themes = []
+    for t in themes:
+        key = t["summary"][:50]
+        if key not in seen_summaries:
+            seen_summaries.add(key)
+            deduped_themes.append(t)
+    themes = deduped_themes[:5]
     
     # Extract timeline events from findings (dates in titles/descriptions)
     timeline_events: list[dict] = []

{osintengine-1.0.2 → osintengine-1.0.4}/src/watson/pipeline/pre_synthesis.py

@@ -501,7 +501,7 @@ def relevance_score(profile: TargetProfile, text: str) -> float:
 
     # ── Strategy 2.5: Surname-aware penalty (person targets only) ──
     # For multi-word person names, a finding that matches only ONE name part
-    # (e.g., "Ildiko Szabo" matching only "Ildiko" from "BÓDI Ildikó") is a
+    # (e.g., "Anna Kovacs" matching only "Anna" from "KOVÁCS Anna") is a
     # completely different person. Given names are too common; the surname is
     # the primary disambiguator. This penalty prevents the trigram/Jaccard
     # overlap from giving high scores to clearly irrelevant findings.