voidaccess 1.9.1__tar.gz → 1.9.3__tar.gz

{voidaccess-1.9.1 → voidaccess-1.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.1
+Version: 1.9.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -172,6 +172,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
 
 Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
 
+### Optional vector embeddings
+
+The default installation can run without PyTorch. When the embedding stack is
+unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
+encoder. Install the optional NLP dependencies to enable full
+sentence-transformer vector embeddings:
+
+```bash
+pip install "voidaccess[nlp]"
+```
+
 ## LLM Providers
 
 | Provider | Typical models | Notes |

{voidaccess-1.9.1 → voidaccess-1.9.3}/README.md

@@ -87,6 +87,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
 
 Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
 
+### Optional vector embeddings
+
+The default installation can run without PyTorch. When the embedding stack is
+unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
+encoder. Install the optional NLP dependencies to enable full
+sentence-transformer vector embeddings:
+
+```bash
+pip install "voidaccess[nlp]"
+```
+
 ## LLM Providers
 
 | Provider | Typical models | Notes |

{voidaccess-1.9.1 → voidaccess-1.9.3}/api/main.py

@@ -40,7 +40,7 @@ logging.basicConfig(level=logging.DEBUG)
 # Rate limiter setup
 # ---------------------------------------------------------------------------
 
-DISABLE_RATE_LIMIT = os.getenv("DISABLE_RATE_LIMIT", "false").lower() == "true"
+DISABLE_RATE_LIMIT = (os.getenv("DISABLE_RATE_LIMIT", "false") or "false").lower() == "true"
 
 if DISABLE_RATE_LIMIT:
     limiter = None

{voidaccess-1.9.1 → voidaccess-1.9.3}/api/routes/investigations.py

@@ -537,7 +537,7 @@ async def _check_cancelled(inv_uuid: uuid.UUID, investigation_id: str) -> bool:
 # Rate limiting (shared key_func with api/main.py; enforcement via app.state.limiter)
 # ---------------------------------------------------------------------------
 
-_DISABLE_RATE_LIMIT = os.getenv("DISABLE_RATE_LIMIT", "false").lower() == "true"
+_DISABLE_RATE_LIMIT = (os.getenv("DISABLE_RATE_LIMIT", "false") or "false").lower() == "true"
 
 if not _DISABLE_RATE_LIMIT:
     try:

{voidaccess-1.9.1 → voidaccess-1.9.3}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.9.1"
+PACKAGE_VERSION = "1.9.3"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.9.1 → voidaccess-1.9.3}/export/misp.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 import json
 import logging
 import os
+import uuid
 from datetime import datetime, timezone
 from typing import Any, Optional
 
@@ -76,6 +77,11 @@ _MISP_ATTR_MAP: dict[str, dict] = {
         "category": "Attribution",
         "to_ids": False,
     },
+    "ORGANIZATION_NAME": {
+        "type": "target-org",
+        "category": "Targeting data",
+        "to_ids": False,
+    },
 }
 
 
@@ -94,7 +100,7 @@ def investigation_to_misp_event(
     Returns a valid (but empty-attribute) event if the investigation is not found.
     Never raises.
     """
-    investigation, entities = _load_investigation_and_entities(
+    investigation, entity_rows = _load_investigation_and_entity_rows(
         investigation_id, entity_ids=entity_ids
     )
 
@@ -110,7 +116,14 @@ def investigation_to_misp_event(
     query = getattr(investigation, "query", "") or ""
 
     attributes: list[dict] = []
-    for entity in entities:
+    graph_endpoint_map: dict[str, str] = {}
+    objects: list[dict] = []
+
+    from extractor.identity import entity_graph_id  # noqa: PLC0415
+    from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+
+    for row in entity_rows:
+        entity = normalized_entity_from_db_row(row)
         mapping = _MISP_ATTR_MAP.get(entity.entity_type)
         if mapping is None:
             continue
@@ -123,6 +136,40 @@ def investigation_to_misp_event(
         }
         attributes.append(attr)
 
+        object_uuid = str(row.id)
+        graph_endpoint_map[entity_graph_id(entity)] = object_uuid
+        objects.append(
+            {
+                "name": "voidaccess-entity",
+                "meta-category": "threat-intelligence",
+                "uuid": object_uuid,
+                "Attribute": [dict(attr, uuid=str(uuid.uuid4()))],
+                "ObjectReference": [],
+            }
+        )
+
+    # Use the same graph relationship enumeration as STIX.  This includes
+    # typed persisted edges and applies the same endpoint/deduplication rules.
+    from export.relationships import load_export_relationships  # noqa: PLC0415
+    graph_relationships, relationship_warning = load_export_relationships(
+        investigation_id, graph_endpoint_map
+    )
+    if relationship_warning:
+        logger.warning("MISP relationships were not included: %s", relationship_warning)
+
+    object_by_uuid = {obj["uuid"]: obj for obj in objects}
+    for relationship in graph_relationships:
+        source_object = object_by_uuid.get(relationship["source_ref"])
+        if source_object is None or relationship["target_ref"] not in object_by_uuid:
+            continue
+        source_object["ObjectReference"].append(
+            {
+                "object_uuid": relationship["source_ref"],
+                "referenced_uuid": relationship["target_ref"],
+                "relationship_type": relationship["edge_type"],
+            }
+        )
+
     return {
         "Event": {
             "info": f"VoidAccess Investigation: {query}",
@@ -131,6 +178,7 @@ def investigation_to_misp_event(
             "analysis": "2",           # Completed
             "distribution": "0",       # Your organisation only
             "Attribute": attributes,
+            "Object": objects,
         }
     }
 
@@ -163,8 +211,21 @@ def _load_investigation_and_entities(
 
     Returns (investigation, entities) or (None, []) on error / not found.
     """
-    import uuid as _uuid
+    investigation, rows = _load_investigation_and_entity_rows(
+        investigation_id, entity_ids=entity_ids
+    )
+    if investigation is None:
+        return None, []
 
+    from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+    return investigation, [normalized_entity_from_db_row(row) for row in rows]
+
+
+def _load_investigation_and_entity_rows(
+    investigation_id: Any,
+    entity_ids: Optional[list[str]] = None,
+):
+    """Load an investigation and its ORM entity rows for export rendering."""
     if not os.getenv("DATABASE_URL"):
         return None, []
 
@@ -172,7 +233,7 @@ def _load_investigation_and_entities(
         from db.session import get_session  # noqa: PLC0415
         from db.queries import get_investigation_by_id_or_run  # noqa: PLC0415
         from db.models import Entity, InvestigationEntityLink  # noqa: PLC0415
-        from extractor.normalizer import NormalizedEntity  # noqa: PLC0415
+        from sqlalchemy.orm import joinedload  # noqa: PLC0415
 
         inv_uuid = _coerce_uuid(investigation_id)
         if inv_uuid is None:
@@ -192,15 +253,16 @@ def _load_investigation_and_entities(
             if investigation is None:
                 return None, []
 
-            linked_ids_select = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id == investigation.id)
-            )
             db_entities = (
                 session.query(Entity)
+                .outerjoin(
+                    InvestigationEntityLink,
+                    InvestigationEntityLink.entity_id == Entity.id,
+                )
+                .options(joinedload(Entity.page))
                 .filter(
                     (Entity.investigation_id == investigation.id)
-                    | Entity.id.in_(linked_ids_select)
+                    | (InvestigationEntityLink.investigation_id == investigation.id)
                 )
                 .all()
             )
@@ -209,17 +271,8 @@ def _load_investigation_and_entities(
                 want = frozenset(filter_uuids)
                 db_entities = [e for e in db_entities if e.id in want]
 
-            from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
-
-            # Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
-            # auxiliary fields (source_quality, extraction_method) no longer
-            # differ between export paths.
-            normalized: list[NormalizedEntity] = [
-                normalized_entity_from_db_row(e) for e in db_entities
-            ]
-
             session.expunge_all()
-            return investigation, normalized
+            return investigation, db_entities
 
     except Exception as exc:
         logger.warning("_load_investigation_and_entities failed: %s", exc)

voidaccess-1.9.3/export/relationships.py

@@ -0,0 +1,81 @@
+"""Shared relationship loading for structured exporters.
+
+The graph builder is the canonical source for both persisted typed edges and
+the bounded co-occurrence edges used by exports.  Keeping the enumeration in
+one place prevents one format from silently losing relationships that another
+format preserves.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+import uuid
+
+from graph.model import RELATIONSHIP_TYPE_STIX
+
+logger = logging.getLogger(__name__)
+
+
+def load_export_relationships(
+    investigation_id: Any,
+    endpoint_map: dict[str, str],
+) -> tuple[list[dict[str, Any]], Optional[str]]:
+    """Return graph relationships whose endpoints exist in ``endpoint_map``.
+
+    Each result contains the original VoidAccess ``edge_type`` plus its STIX
+    spelling.  Consumers can therefore share the exact same deduplication and
+    endpoint selection while rendering format-specific relationship records.
+    """
+    try:
+        from graph.builder import build_graph_from_db  # noqa: PLC0415
+
+        inv_uuid = _coerce_uuid(investigation_id)
+        graph = build_graph_from_db(investigation_id=inv_uuid)
+        relationships: list[dict[str, Any]] = []
+        seen: set[tuple[str, str, str]] = set()
+
+        for source_node, target_node, data in graph.edges(data=True):
+            source_ref = endpoint_map.get(source_node)
+            target_ref = endpoint_map.get(target_node)
+            if not source_ref or not target_ref:
+                continue
+
+            edge_type = _normalise_edge_type(data.get("edge_type", "related-to"))
+            stix_type = RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
+            key = (stix_type, source_ref, target_ref)
+            if key in seen:
+                continue
+            seen.add(key)
+            relationships.append(
+                {
+                    "source_ref": source_ref,
+                    "target_ref": target_ref,
+                    "edge_type": edge_type,
+                    "stix_relationship_type": stix_type,
+                    "confidence": data.get("confidence"),
+                }
+            )
+        return relationships, None
+    except Exception as exc:
+        logger.warning("load_export_relationships failed: %s", exc)
+        return [], f"load_export_relationships failed: {exc}"
+
+
+def _normalise_edge_type(edge_type: Any) -> str:
+    """Normalise enum/string edge types without changing their meaning."""
+    value = getattr(edge_type, "value", edge_type)
+    text = str(value or "related-to").strip()
+    upper = text.upper()
+    if upper in RELATIONSHIP_TYPE_STIX:
+        return upper
+    return text
+
+
+def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
+    if isinstance(value, uuid.UUID):
+        return value
+    try:
+        return uuid.UUID(str(value))
+    except (ValueError, AttributeError):
+        return None

{voidaccess-1.9.1 → voidaccess-1.9.3}/export/stix.py

@@ -18,11 +18,11 @@ from __future__ import annotations
 import json
 import logging
 import os
+import re
 import warnings
 from typing import Any, Optional
 import uuid
 
-from graph.model import RELATIONSHIP_TYPE_STIX
 from extractor.confidence import get_entity_confidence
 
 logger = logging.getLogger(__name__)
@@ -79,6 +79,114 @@ _STIX_PATTERNS: dict[str, str] = {
 # Entity types that map to STIX Malware objects
 _MALWARE_TYPES = frozenset({"MALWARE_FAMILY", "RANSOMWARE_GROUP"})
 
+# STIX 2.1's malware-type vocabulary is intentionally small.  Keep the
+# mapping here instead of treating every MALWARE_FAMILY as the generic
+# ``malware`` type: family names and enrichment metadata often carry enough
+# information for SIEM consumers to filter useful categories.
+_STIX_MALWARE_TYPE_ALIASES = {
+    "infostealer": "spyware",
+    "stealer": "spyware",
+    "remote_access_trojan": "remote-access-trojan",
+    "remote access trojan": "remote-access-trojan",
+    "rat": "remote-access-trojan",
+    "coinminer": "resource-exploitation",
+    "cryptominer": "resource-exploitation",
+    "crypto-miner": "resource-exploitation",
+    "miner": "resource-exploitation",
+}
+_STIX_MALWARE_TYPES = frozenset({
+    "adware",
+    "backdoor",
+    "bot",
+    "ddos",
+    "dropper",
+    "exploit-kit",
+    "keylogger",
+    "ransomware",
+    "remote-access-trojan",
+    "resource-exploitation",
+    "rootkit",
+    "screen-capture",
+    "spyware",
+    "wiper",
+    "malware",
+})
+
+# These are conservative, high-signal family/name hints.  Unknown families
+# retain the valid generic STIX type rather than being assigned a speculative
+# category.
+_MALWARE_NAME_PATTERNS = (
+    ("ransomware", re.compile(
+        r"\b(?:ransomware|lockbit|blackcat|alphv|clop|conti|ryuk|hive|revil|darkside|wannacry)\b",
+        re.IGNORECASE,
+    )),
+    ("remote-access-trojan", re.compile(
+        r"\b(?:remote\s+access\s+trojan|remcos|njrat|nanocore|quasar|darkcomet|xrat)\b",
+        re.IGNORECASE,
+    )),
+    ("spyware", re.compile(
+        r"\b(?:spyware|infostealer|stealer|redline|vidar|lumma|raccoon|agent\s+tesla|formbook)\b",
+        re.IGNORECASE,
+    )),
+    ("bot", re.compile(
+        r"\b(?:botnet|emotet|qakbot|qbot|trickbot|mirai|zeus|mozi)\b",
+        re.IGNORECASE,
+    )),
+    ("keylogger", re.compile(r"\bkeylogger\b", re.IGNORECASE)),
+    ("exploit-kit", re.compile(r"\bexploit[\s-]+kit\b", re.IGNORECASE)),
+    ("rootkit", re.compile(r"\brootkit\b", re.IGNORECASE)),
+    ("screen-capture", re.compile(r"\bscreen[\s-]+capture\b", re.IGNORECASE)),
+    ("dropper", re.compile(r"\bdropper\b", re.IGNORECASE)),
+    ("wiper", re.compile(r"\bwiper\b", re.IGNORECASE)),
+    ("adware", re.compile(r"\badware\b", re.IGNORECASE)),
+    ("resource-exploitation", re.compile(
+        r"\b(?:cryptominer|crypto[\s-]+miner|coinminer|cryptojacking)\b",
+        re.IGNORECASE,
+    )),
+    ("backdoor", re.compile(r"\bbackdoor\b", re.IGNORECASE)),
+)
+
+
+def _entity_attribute(entity: Any, name: str) -> Any:
+    """Read optional classification metadata from dicts and ORM-like rows."""
+    if isinstance(entity, dict):
+        return entity.get(name)
+    return getattr(entity, name, None)
+
+
+def _normalise_malware_type(value: Any) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    key = value.strip().lower().replace("_", " ")
+    key = _STIX_MALWARE_TYPE_ALIASES.get(key, key)
+    key = key.replace(" ", "-")
+    return key if key in _STIX_MALWARE_TYPES else None
+
+
+def _classify_malware(entity: Any) -> list[str]:
+    """Return evidence-backed STIX malware types for an entity."""
+    entity_type = str(getattr(entity, "entity_type", "") or "").upper()
+    if isinstance(entity, dict):
+        entity_type = str(entity.get("entity_type") or "").upper()
+    if entity_type == "RANSOMWARE_GROUP":
+        return ["ransomware"]
+
+    for field_name in ("malware_types", "malware_type", "malware_category", "category", "family_type"):
+        raw = _entity_attribute(entity, field_name)
+        values = raw if isinstance(raw, (list, tuple, set)) else [raw]
+        classified = [item for item in (_normalise_malware_type(v) for v in values) if item]
+        if classified:
+            return list(dict.fromkeys(classified))
+
+    evidence = " ".join(
+        str(_entity_attribute(entity, field) or "")
+        for field in ("value", "context_snippet")
+    )
+    for malware_type, pattern in _MALWARE_NAME_PATTERNS:
+        if pattern.search(evidence):
+            return [malware_type]
+    return ["malware"]
+
 # ---------------------------------------------------------------------------
 # Confidence mapping: VoidAccess float → STIX integer (0-100)
 # ---------------------------------------------------------------------------
@@ -161,6 +269,7 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         malware = stix2.Malware(
             name=entity.value,
             is_family=True,
+            malware_types=_classify_malware(entity),
             confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
@@ -176,6 +285,28 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         return None
 
 
+def entity_to_stix_identity(entity: Any) -> Optional[Any]:
+    """Convert an organization entity to a STIX Identity object."""
+    if not _STIX2_AVAILABLE or entity.entity_type != "ORGANIZATION_NAME":
+        return None
+    try:
+        return stix2.Identity(
+            name=entity.value,
+            identity_class="organization",
+            confidence=_to_stix_confidence(get_entity_confidence(entity)),
+            allow_custom=True,
+            x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
+            external_references=(
+                [{"source_name": "voidaccess", "url": entity.source_url}]
+                if entity.source_url
+                else []
+            ),
+        )
+    except Exception as exc:
+        logger.warning("entity_to_stix_identity failed for %r: %s", entity.value, exc)
+        return None
+
+
 def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
     """
     Convert a THREAT_ACTOR_HANDLE entity to a STIX 2.1 ThreatActor object.
@@ -287,6 +418,11 @@ def investigation_to_stix_bundle(
             stix_objects.append(actor)
             _register(entity, actor.id)
 
+        identity = entity_to_stix_identity(entity)
+        if identity:
+            stix_objects.append(identity)
+            _register(entity, identity.id)
+
     if include_relationships and stix_objects:
         relationships, relationship_warning = _build_stix_relationships(
             investigation_id, stix_id_map
@@ -436,32 +572,22 @@ def _build_stix_relationships(
     if not _STIX2_AVAILABLE:
         return [], "stix2 is not installed"
     try:
-        from graph.builder import build_graph_from_db  # noqa: PLC0415
-
-        inv_uuid = _coerce_uuid(investigation_id)
-        graph = build_graph_from_db(investigation_id=inv_uuid)
+        from export.relationships import load_export_relationships  # noqa: PLC0415
 
+        graph_relationships, warning = load_export_relationships(
+            investigation_id, stix_id_map
+        )
+        if warning:
+            return [], warning
         relationships: list[Any] = []
-        seen_relationships: set[tuple[str, str, str]] = set()
-        for source_node, target_node, data in graph.edges(data=True):
-            src_stix_id = stix_id_map.get(source_node)
-            tgt_stix_id = stix_id_map.get(target_node)
-            if not src_stix_id or not tgt_stix_id:
-                continue
-            edge_type = data.get("edge_type", "related-to")
-            # Map VoidAccess edge types to STIX relationship types
-            rel_type = _edge_type_to_stix(edge_type)
-            relationship_key = (rel_type, src_stix_id, tgt_stix_id)
-            if relationship_key in seen_relationships:
-                continue
-            seen_relationships.add(relationship_key)
+        for data in graph_relationships:
             # Carry the edge's own confidence onto the SRO.  Typed relationships
             # have a claim-specific confidence distinct from co-occurrence's
             # flat value; STIX 2.1 confidence is an int 0-100.
             rel_kwargs: dict[str, Any] = {
-                "relationship_type": rel_type,
-                "source_ref": src_stix_id,
-                "target_ref": tgt_stix_id,
+                "relationship_type": data["stix_relationship_type"],
+                "source_ref": data["source_ref"],
+                "target_ref": data["target_ref"],
                 "allow_custom": True,
             }
             try:
@@ -488,7 +614,9 @@ def _build_stix_relationships(
 
 def _edge_type_to_stix(edge_type: str) -> str:
     """Map VoidAccess graph edge types to STIX relationship type strings."""
-    return RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
+    from export.relationships import _normalise_edge_type  # noqa: PLC0415
+    from graph.model import RELATIONSHIP_TYPE_STIX  # noqa: PLC0415
+    return RELATIONSHIP_TYPE_STIX.get(_normalise_edge_type(edge_type), "related-to")
 
 
 def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:

{voidaccess-1.9.1 → voidaccess-1.9.3}/graph/builder.py

@@ -789,8 +789,12 @@ def build_graph_from_db(
                     .join(
                         InvestigationEntityLink,
                         InvestigationEntityLink.entity_id == Entity.id,
+                        isouter=True,
+                    )
+                    .filter(
+                        (Entity.investigation_id == investigation_id)
+                        | (InvestigationEntityLink.investigation_id == investigation_id)
                     )
-                    .filter(InvestigationEntityLink.investigation_id == investigation_id)
                     .options(joinedload(Entity.page))
                 )
             else:
@@ -1230,8 +1234,12 @@ def build_graph_from_db_cached(investigation_id: uuid.UUID) -> nx.MultiDiGraph:
             .join(
                 InvestigationEntityLink,
                 InvestigationEntityLink.entity_id == Entity.id,
+                isouter=True,
+            )
+            .filter(
+                (Entity.investigation_id == investigation_id)
+                | (InvestigationEntityLink.investigation_id == investigation_id)
             )
-            .filter(InvestigationEntityLink.investigation_id == investigation_id)
             .options(joinedload(Entity.page))
             .yield_per(500)
         )

{voidaccess-1.9.1 → voidaccess-1.9.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.9.1"
+version = "1.9.3"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"

{voidaccess-1.9.1 → voidaccess-1.9.3}/sources/email_reputation.py

@@ -555,8 +555,10 @@ def _update_email_entities_in_db(
                             existing.append(tag)
                     db_entity.corroborating_sources = json.dumps(existing)
             session.commit()
-    except Exception as exc:
-        logger.warning("email_reputation: DB update failed: %s", exc)
+    except Exception:
+        # Do not log the raw SQLAlchemy exception: its parameter dump can
+        # contain the full email address being enriched.
+        logger.warning("email_reputation: DB update failed")
 
 
 # ---------------------------------------------------------------------------