voidaccess 1.8.1__tar.gz → 1.8.2__tar.gz

{voidaccess-1.8.1/voidaccess.egg-info → voidaccess-1.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.8.1
+Version: 1.8.2
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-1.8.1 → voidaccess-1.8.2}/export/ioc_package.py

@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.8.1"
+PACKAGE_VERSION = "1.8.2"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.8.1 → voidaccess-1.8.2}/extractor/pipeline.py

@@ -574,6 +574,23 @@ async def extract_entities_from_pages(
                     )
                     page_id_by_url[_url] = _p.get("page_id")
 
+                # Some clearnet/side-source pages are materialized in the DB
+                # during entity persistence, after the CLI's initial page-id
+                # lookup.  Resolve those URLs again before relationship
+                # extraction so provenance is never lost to that ordering.
+                missing_urls = [url for url, page_id in page_id_by_url.items() if page_id is None]
+                if missing_urls:
+                    from db.models import Page  # noqa: PLC0415
+
+                    with get_session() as _page_session:
+                        persisted_pages = (
+                            _page_session.query(Page)
+                            .filter(Page.url.in_(missing_urls))
+                            .all()
+                        )
+                        for _page in persisted_pages:
+                            page_id_by_url[_page.url] = _page.id
+
                 max_rel_pages = int(
                     getattr(_config, "MAX_REL_PAGES_PER_INV", 10) or 10
                 )

{voidaccess-1.8.1 → voidaccess-1.8.2}/extractor/relationship_extract.py

@@ -76,6 +76,77 @@ _PASSIVE_REL_VOCAB: dict[str, str] = {
     "controlled_by": "CONTROLS",
 }
 
+# Endpoint-type validation is deliberately kept in the relationship extractor
+# so the prompt and the independent post-LLM gate share one source of truth.
+# The sets are intentionally permissive for the project's current normalized
+# entity vocabulary; unknown types are not accepted for typed edges.
+_RELATIONSHIP_TYPE_COMPATIBILITY: dict[str, tuple[set[str], set[str]]] = {
+    "USES": (
+        {"RANSOMWARE_GROUP", "THREAT_ACTOR", "THREAT_ACTOR_HANDLE"},
+        {"MALWARE_FAMILY", "MALWARE", "TOOL", "SOFTWARE"},
+    ),
+    "DROPS": (
+        {"MALWARE_FAMILY", "MALWARE"},
+        {
+            "MALWARE_FAMILY", "MALWARE", "FILE_HASH_MD5", "FILE_HASH_SHA1",
+            "FILE_HASH_SHA256",
+        },
+    ),
+    "CONTROLS": (
+        {"RANSOMWARE_GROUP", "THREAT_ACTOR", "THREAT_ACTOR_HANDLE"},
+        {
+            "CRYPTO_WALLET", "BITCOIN_ADDRESS", "MONERO_ADDRESS", "ETH_ADDRESS",
+            "ETHEREUM_ADDRESS", "LITECOIN_ADDRESS", "ZCASH_ADDRESS",
+            "DOGECOIN_ADDRESS", "XRP_ADDRESS", "SOLANA_ADDRESS", "TRON_ADDRESS",
+            "BITCOIN_CASH_ADDRESS", "DASH_ADDRESS", "IP_ADDRESS", "IPV6_ADDRESS",
+            "DOMAIN", "ENS_DOMAIN", "ONION_URL",
+        },
+    ),
+    "TARGETS": (
+        {"RANSOMWARE_GROUP", "THREAT_ACTOR", "THREAT_ACTOR_HANDLE", "MALWARE_FAMILY", "MALWARE"},
+        {"ORGANIZATION_NAME"},
+    ),
+    "EXPLOITS": (
+        {"RANSOMWARE_GROUP", "THREAT_ACTOR", "THREAT_ACTOR_HANDLE", "MALWARE_FAMILY", "MALWARE"},
+        {"CVE", "CVE_NUMBER", "EXPLOIT_DB_ID"},
+    ),
+    "COMMUNICATES_WITH": (
+        {"RANSOMWARE_GROUP", "THREAT_ACTOR", "THREAT_ACTOR_HANDLE", "MALWARE_FAMILY", "MALWARE"},
+        {
+            "IP_ADDRESS", "IPV6_ADDRESS", "DOMAIN", "ENS_DOMAIN", "ONION_URL",
+            "TELEGRAM_HANDLE", "DISCORD_HANDLE", "XMPP_JID", "TOX_ID",
+            "MATRIX_HANDLE", "WIRE_HANDLE", "ICQ_NUMBER", "WICKR_ID",
+            "PHONE_NUMBER", "EMAIL_ADDRESS",
+        },
+    ),
+}
+
+
+def _entity_type(entity: dict) -> str:
+    return str(entity.get("type", "") or "").strip().upper()
+
+
+def _is_compatible_relationship(rel_type: str, source: dict, target: dict) -> bool:
+    """Validate endpoint types independently of the LLM's claimed verb."""
+    allowed = _RELATIONSHIP_TYPE_COMPATIBILITY.get(rel_type)
+    if allowed is None:
+        return False
+    source_types, target_types = allowed
+    return _entity_type(source) in source_types and _entity_type(target) in target_types
+
+
+def _compatibility_guide() -> str:
+    lines = []
+    for rel_type, (sources, targets) in _RELATIONSHIP_TYPE_COMPATIBILITY.items():
+        lines.append(
+            f"- {rel_type}: source {{{', '.join(sorted(sources))}}} -> "
+            f"target {{{', '.join(sorted(targets))}}}"
+        )
+    return "\n".join(lines)
+
+
+_COMPATIBILITY_GUIDE = _compatibility_guide()
+
 _PROMPT_TEMPLATE = (
     "You are a threat intelligence analyst. Below is a numbered list of "
     "entities that were already extracted from a single dark-web page, "
@@ -100,6 +171,9 @@ _PROMPT_TEMPLATE = (
     "Plain co-occurrence is captured elsewhere — omit pairs that have no "
     "specific relationship.\n"
     "- Do not output any text outside the JSON object.\n\n"
+    "Entity-type compatibility guide (source -> target):\n"
+    "{compatibility_guide}\n"
+    "Only emit a relationship when BOTH endpoint types match this guide.\n\n"
     "Entities:\n{entities}\n\n"
     "Content:\n{content}"
 )
@@ -216,7 +290,11 @@ async def extract_relationships_for_page(
         for i, e in enumerate(trimmed)
     )
     content = page_text[:max_chunk_chars]
-    prompt = _PROMPT_TEMPLATE.format(entities=entity_lines, content=content)
+    prompt = _PROMPT_TEMPLATE.format(
+        entities=entity_lines,
+        content=content,
+        compatibility_guide=_COMPATIBILITY_GUIDE,
+    )
 
     try:
         raw = await _invoke_llm(prompt, llm)
@@ -263,6 +341,12 @@ async def extract_relationships_for_page(
             # its plain co-occurrence edge, generated elsewhere.
             continue
 
+        if not _is_compatible_relationship(rel_type, a_ent, b_ent):
+            # Structural validation is independent of the model's claim.  Do
+            # not downgrade invalid pairs to co-occurrence: the pair may be
+            # spurious even though both entities appeared on the page.
+            continue
+
         a_id = a_ent.get("id")
         b_id = b_ent.get("id")
         if not a_id or not b_id or a_id == b_id:

{voidaccess-1.8.1 → voidaccess-1.8.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.8.1"
+version = "1.8.2"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"

voidaccess-1.8.2/tests/test_relationship_extract.py

@@ -0,0 +1,74 @@
+import pytest
+
+from extractor.relationship_extract import (
+    _COMPATIBILITY_GUIDE,
+    extract_relationships_for_page,
+    extract_relationships_from_results,
+)
+
+
+class _Response:
+    content = '{"relationships": [{"source": 0, "target": 1, "type": "uses", "confidence": 0.9}, {"source": 0, "target": 2, "type": "uses", "confidence": 0.9}]}'
+
+
+class _LLM:
+    def __init__(self):
+        self.prompt = ""
+
+    def with_config(self, _config):
+        return self
+
+    async def ainvoke(self, prompt):
+        self.prompt = prompt
+        return _Response()
+
+
+@pytest.mark.asyncio
+async def test_prompt_contains_shared_compatibility_guide_and_invalid_pair_is_dropped():
+    llm = _LLM()
+    entities = [
+        {"id": "actor", "type": "THREAT_ACTOR_HANDLE", "value": "@actor"},
+        {"id": "tool", "type": "TOOL", "value": "Rclone"},
+        {"id": "hash", "type": "FILE_HASH_SHA256", "value": "abc"},
+    ]
+
+    relationships = await extract_relationships_for_page("actor uses rclone", entities, llm)
+
+    assert relationships == [
+        {
+            "entity_a_id": "actor",
+            "entity_b_id": "tool",
+            "relationship_type": "USES",
+            "confidence": 0.9,
+        }
+    ]
+    assert _COMPATIBILITY_GUIDE in llm.prompt
+
+
+@pytest.mark.asyncio
+async def test_relationship_provenance_uses_page_manifest_id():
+    class _Result:
+        page_url = "https://example.test/page"
+        entity_ids = ["actor", "org"]
+        entities = [
+            type("Entity", (), {"entity_type": "THREAT_ACTOR_HANDLE", "value": "@actor", "confidence": 1.0})(),
+            type("Entity", (), {"entity_type": "ORGANIZATION_NAME", "value": "Acme", "confidence": 1.0})(),
+        ]
+
+    class _TargetLLM(_LLM):
+        async def ainvoke(self, prompt):
+            self.prompt = prompt
+            return type(
+                "Response",
+                (),
+                {"content": '{"relationships": [{"source": 0, "target": 1, "type": "targets", "confidence": 0.8}]}'},
+            )()
+
+    relationships = await extract_relationships_from_results(
+        [_Result()],
+        {"https://example.test/page": "actor targets Acme"},
+        {"https://example.test/page": "page-123"},
+        _TargetLLM(),
+    )
+
+    assert relationships[0]["source_page_id"] == "page-123"

{voidaccess-1.8.1 → voidaccess-1.8.2/voidaccess.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.8.1
+Version: 1.8.2
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-1.8.1 → voidaccess-1.8.2}/voidaccess.egg-info/SOURCES.txt

@@ -171,6 +171,7 @@ tests/test_paste_scraper.py
 tests/test_proxy_client.py
 tests/test_ransomlook.py
 tests/test_regex_patterns.py
+tests/test_relationship_extract.py
 tests/test_rss_scraper.py
 tests/test_scrape_js.py
 tests/test_settings.py

{voidaccess-1.8.1 → voidaccess-1.8.2}/voidaccess_cli/__init__.py

@@ -1,3 +1,3 @@
 """voidaccess CLI — dark-web OSINT command-line interface."""
 
-__version__ = "1.8.1"
+__version__ = "1.8.2"

{voidaccess-1.8.1 → voidaccess-1.8.2}/voidaccess_cli/commands/investigate.py

@@ -915,6 +915,17 @@ async def _run_investigation(
     display.update_step("Finalizing results", "active")
     final_entities = sqlite_adapter.get_entities(investigation_id)
     final_relationships = sqlite_adapter.get_relationships(investigation_id)
+    # Entity persistence may create Page rows for side-source pages after the
+    # initial pre-extraction lookup. Refresh the lightweight manifest so it
+    # agrees with relationship provenance and the persisted DB state.
+    refreshed_page_ids = await asyncio.to_thread(
+        _lookup_page_ids, [p["url"] for p in scraped_pages]
+    )
+    for page in scraped_pages:
+        if page.get("page_id") is None:
+            page_id = refreshed_page_ids.get(page["url"])
+            if page_id is not None:
+                page["page_id"] = page_id
     sqlite_adapter.update_investigation(
         investigation_id,
         {
@@ -958,7 +969,18 @@ async def _run_investigation(
         "sources_used": sources_used,
         "entities": final_entities,
         "relationships": final_relationships,
-        "pages_scraped": [{"url": p["url"], "source": p.get("source", "")} for p in scraped_pages],
+        # Keep the lightweight output manifest needed to resolve relationship
+        # provenance.  Text is intentionally omitted; the DB remains the
+        # source of page content while each relationship can point to a real
+        # page in this investigation.
+        "pages_scraped": [
+            {
+                "page_id": p.get("page_id"),
+                "url": p["url"],
+                "source": p.get("source", ""),
+            }
+            for p in scraped_pages
+        ],
         "communities": communities,
         "community_count": len(set(communities.values())) if communities else 0,
         "seeds_discovered": _discovered_count,