voidaccess 1.9.1__tar.gz → 1.9.2__tar.gz

{voidaccess-1.9.1 → voidaccess-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.1
+Version: 1.9.2
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -172,6 +172,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
 
 Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
 
+### Optional vector embeddings
+
+The default installation can run without PyTorch. When the embedding stack is
+unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
+encoder. Install the optional NLP dependencies to enable full
+sentence-transformer vector embeddings:
+
+```bash
+pip install "voidaccess[nlp]"
+```
+
 ## LLM Providers
 
 | Provider | Typical models | Notes |

{voidaccess-1.9.1 → voidaccess-1.9.2}/README.md

@@ -87,6 +87,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
 
 Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
 
+### Optional vector embeddings
+
+The default installation can run without PyTorch. When the embedding stack is
+unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
+encoder. Install the optional NLP dependencies to enable full
+sentence-transformer vector embeddings:
+
+```bash
+pip install "voidaccess[nlp]"
+```
+
 ## LLM Providers
 
 | Provider | Typical models | Notes |

{voidaccess-1.9.1 → voidaccess-1.9.2}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.9.1"
+PACKAGE_VERSION = "1.9.2"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.9.1 → voidaccess-1.9.2}/export/misp.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 import json
 import logging
 import os
+import uuid
 from datetime import datetime, timezone
 from typing import Any, Optional
 
@@ -76,6 +77,11 @@ _MISP_ATTR_MAP: dict[str, dict] = {
         "category": "Attribution",
         "to_ids": False,
     },
+    "ORGANIZATION_NAME": {
+        "type": "target-org",
+        "category": "Targeting data",
+        "to_ids": False,
+    },
 }
 
 
@@ -94,7 +100,7 @@ def investigation_to_misp_event(
     Returns a valid (but empty-attribute) event if the investigation is not found.
     Never raises.
     """
-    investigation, entities = _load_investigation_and_entities(
+    investigation, entity_rows = _load_investigation_and_entity_rows(
         investigation_id, entity_ids=entity_ids
     )
 
@@ -110,7 +116,14 @@ def investigation_to_misp_event(
     query = getattr(investigation, "query", "") or ""
 
     attributes: list[dict] = []
-    for entity in entities:
+    graph_endpoint_map: dict[str, str] = {}
+    objects: list[dict] = []
+
+    from extractor.identity import entity_graph_id  # noqa: PLC0415
+    from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+
+    for row in entity_rows:
+        entity = normalized_entity_from_db_row(row)
         mapping = _MISP_ATTR_MAP.get(entity.entity_type)
         if mapping is None:
             continue
@@ -123,6 +136,40 @@ def investigation_to_misp_event(
         }
         attributes.append(attr)
 
+        object_uuid = str(row.id)
+        graph_endpoint_map[entity_graph_id(entity)] = object_uuid
+        objects.append(
+            {
+                "name": "voidaccess-entity",
+                "meta-category": "threat-intelligence",
+                "uuid": object_uuid,
+                "Attribute": [dict(attr, uuid=str(uuid.uuid4()))],
+                "ObjectReference": [],
+            }
+        )
+
+    # Use the same graph relationship enumeration as STIX.  This includes
+    # typed persisted edges and applies the same endpoint/deduplication rules.
+    from export.relationships import load_export_relationships  # noqa: PLC0415
+    graph_relationships, relationship_warning = load_export_relationships(
+        investigation_id, graph_endpoint_map
+    )
+    if relationship_warning:
+        logger.warning("MISP relationships were not included: %s", relationship_warning)
+
+    object_by_uuid = {obj["uuid"]: obj for obj in objects}
+    for relationship in graph_relationships:
+        source_object = object_by_uuid.get(relationship["source_ref"])
+        if source_object is None or relationship["target_ref"] not in object_by_uuid:
+            continue
+        source_object["ObjectReference"].append(
+            {
+                "object_uuid": relationship["source_ref"],
+                "referenced_uuid": relationship["target_ref"],
+                "relationship_type": relationship["edge_type"],
+            }
+        )
+
     return {
         "Event": {
             "info": f"VoidAccess Investigation: {query}",
@@ -131,6 +178,7 @@ def investigation_to_misp_event(
             "analysis": "2",           # Completed
             "distribution": "0",       # Your organisation only
             "Attribute": attributes,
+            "Object": objects,
         }
     }
 
@@ -163,8 +211,21 @@ def _load_investigation_and_entities(
 
     Returns (investigation, entities) or (None, []) on error / not found.
     """
-    import uuid as _uuid
+    investigation, rows = _load_investigation_and_entity_rows(
+        investigation_id, entity_ids=entity_ids
+    )
+    if investigation is None:
+        return None, []
 
+    from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+    return investigation, [normalized_entity_from_db_row(row) for row in rows]
+
+
+def _load_investigation_and_entity_rows(
+    investigation_id: Any,
+    entity_ids: Optional[list[str]] = None,
+):
+    """Load an investigation and its ORM entity rows for export rendering."""
     if not os.getenv("DATABASE_URL"):
         return None, []
 
@@ -172,7 +233,7 @@ def _load_investigation_and_entities(
         from db.session import get_session  # noqa: PLC0415
         from db.queries import get_investigation_by_id_or_run  # noqa: PLC0415
         from db.models import Entity, InvestigationEntityLink  # noqa: PLC0415
-        from extractor.normalizer import NormalizedEntity  # noqa: PLC0415
+        from sqlalchemy.orm import joinedload  # noqa: PLC0415
 
         inv_uuid = _coerce_uuid(investigation_id)
         if inv_uuid is None:
@@ -192,15 +253,16 @@ def _load_investigation_and_entities(
             if investigation is None:
                 return None, []
 
-            linked_ids_select = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id == investigation.id)
-            )
             db_entities = (
                 session.query(Entity)
+                .outerjoin(
+                    InvestigationEntityLink,
+                    InvestigationEntityLink.entity_id == Entity.id,
+                )
+                .options(joinedload(Entity.page))
                 .filter(
                     (Entity.investigation_id == investigation.id)
-                    | Entity.id.in_(linked_ids_select)
+                    | (InvestigationEntityLink.investigation_id == investigation.id)
                 )
                 .all()
             )
@@ -209,17 +271,8 @@ def _load_investigation_and_entities(
                 want = frozenset(filter_uuids)
                 db_entities = [e for e in db_entities if e.id in want]
 
-            from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
-
-            # Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
-            # auxiliary fields (source_quality, extraction_method) no longer
-            # differ between export paths.
-            normalized: list[NormalizedEntity] = [
-                normalized_entity_from_db_row(e) for e in db_entities
-            ]
-
             session.expunge_all()
-            return investigation, normalized
+            return investigation, db_entities
 
     except Exception as exc:
         logger.warning("_load_investigation_and_entities failed: %s", exc)

voidaccess-1.9.2/export/relationships.py

@@ -0,0 +1,81 @@
+"""Shared relationship loading for structured exporters.
+
+The graph builder is the canonical source for both persisted typed edges and
+the bounded co-occurrence edges used by exports.  Keeping the enumeration in
+one place prevents one format from silently losing relationships that another
+format preserves.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+import uuid
+
+from graph.model import RELATIONSHIP_TYPE_STIX
+
+logger = logging.getLogger(__name__)
+
+
+def load_export_relationships(
+    investigation_id: Any,
+    endpoint_map: dict[str, str],
+) -> tuple[list[dict[str, Any]], Optional[str]]:
+    """Return graph relationships whose endpoints exist in ``endpoint_map``.
+
+    Each result contains the original VoidAccess ``edge_type`` plus its STIX
+    spelling.  Consumers can therefore share the exact same deduplication and
+    endpoint selection while rendering format-specific relationship records.
+    """
+    try:
+        from graph.builder import build_graph_from_db  # noqa: PLC0415
+
+        inv_uuid = _coerce_uuid(investigation_id)
+        graph = build_graph_from_db(investigation_id=inv_uuid)
+        relationships: list[dict[str, Any]] = []
+        seen: set[tuple[str, str, str]] = set()
+
+        for source_node, target_node, data in graph.edges(data=True):
+            source_ref = endpoint_map.get(source_node)
+            target_ref = endpoint_map.get(target_node)
+            if not source_ref or not target_ref:
+                continue
+
+            edge_type = _normalise_edge_type(data.get("edge_type", "related-to"))
+            stix_type = RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
+            key = (stix_type, source_ref, target_ref)
+            if key in seen:
+                continue
+            seen.add(key)
+            relationships.append(
+                {
+                    "source_ref": source_ref,
+                    "target_ref": target_ref,
+                    "edge_type": edge_type,
+                    "stix_relationship_type": stix_type,
+                    "confidence": data.get("confidence"),
+                }
+            )
+        return relationships, None
+    except Exception as exc:
+        logger.warning("load_export_relationships failed: %s", exc)
+        return [], f"load_export_relationships failed: {exc}"
+
+
+def _normalise_edge_type(edge_type: Any) -> str:
+    """Normalise enum/string edge types without changing their meaning."""
+    value = getattr(edge_type, "value", edge_type)
+    text = str(value or "related-to").strip()
+    upper = text.upper()
+    if upper in RELATIONSHIP_TYPE_STIX:
+        return upper
+    return text
+
+
+def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
+    if isinstance(value, uuid.UUID):
+        return value
+    try:
+        return uuid.UUID(str(value))
+    except (ValueError, AttributeError):
+        return None

{voidaccess-1.9.1 → voidaccess-1.9.2}/export/stix.py

@@ -22,7 +22,6 @@ import warnings
 from typing import Any, Optional
 import uuid
 
-from graph.model import RELATIONSHIP_TYPE_STIX
 from extractor.confidence import get_entity_confidence
 
 logger = logging.getLogger(__name__)
@@ -161,6 +160,11 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         malware = stix2.Malware(
             name=entity.value,
             is_family=True,
+            malware_types=(
+                ["ransomware"]
+                if entity.entity_type == "RANSOMWARE_GROUP"
+                else ["malware"]
+            ),
             confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
@@ -176,6 +180,28 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         return None
 
 
+def entity_to_stix_identity(entity: Any) -> Optional[Any]:
+    """Convert an organization entity to a STIX Identity object."""
+    if not _STIX2_AVAILABLE or entity.entity_type != "ORGANIZATION_NAME":
+        return None
+    try:
+        return stix2.Identity(
+            name=entity.value,
+            identity_class="organization",
+            confidence=_to_stix_confidence(get_entity_confidence(entity)),
+            allow_custom=True,
+            x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
+            external_references=(
+                [{"source_name": "voidaccess", "url": entity.source_url}]
+                if entity.source_url
+                else []
+            ),
+        )
+    except Exception as exc:
+        logger.warning("entity_to_stix_identity failed for %r: %s", entity.value, exc)
+        return None
+
+
 def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
     """
     Convert a THREAT_ACTOR_HANDLE entity to a STIX 2.1 ThreatActor object.
@@ -287,6 +313,11 @@ def investigation_to_stix_bundle(
             stix_objects.append(actor)
             _register(entity, actor.id)
 
+        identity = entity_to_stix_identity(entity)
+        if identity:
+            stix_objects.append(identity)
+            _register(entity, identity.id)
+
     if include_relationships and stix_objects:
         relationships, relationship_warning = _build_stix_relationships(
             investigation_id, stix_id_map
@@ -436,32 +467,22 @@ def _build_stix_relationships(
     if not _STIX2_AVAILABLE:
         return [], "stix2 is not installed"
     try:
-        from graph.builder import build_graph_from_db  # noqa: PLC0415
-
-        inv_uuid = _coerce_uuid(investigation_id)
-        graph = build_graph_from_db(investigation_id=inv_uuid)
+        from export.relationships import load_export_relationships  # noqa: PLC0415
 
+        graph_relationships, warning = load_export_relationships(
+            investigation_id, stix_id_map
+        )
+        if warning:
+            return [], warning
         relationships: list[Any] = []
-        seen_relationships: set[tuple[str, str, str]] = set()
-        for source_node, target_node, data in graph.edges(data=True):
-            src_stix_id = stix_id_map.get(source_node)
-            tgt_stix_id = stix_id_map.get(target_node)
-            if not src_stix_id or not tgt_stix_id:
-                continue
-            edge_type = data.get("edge_type", "related-to")
-            # Map VoidAccess edge types to STIX relationship types
-            rel_type = _edge_type_to_stix(edge_type)
-            relationship_key = (rel_type, src_stix_id, tgt_stix_id)
-            if relationship_key in seen_relationships:
-                continue
-            seen_relationships.add(relationship_key)
+        for data in graph_relationships:
             # Carry the edge's own confidence onto the SRO.  Typed relationships
             # have a claim-specific confidence distinct from co-occurrence's
             # flat value; STIX 2.1 confidence is an int 0-100.
             rel_kwargs: dict[str, Any] = {
-                "relationship_type": rel_type,
-                "source_ref": src_stix_id,
-                "target_ref": tgt_stix_id,
+                "relationship_type": data["stix_relationship_type"],
+                "source_ref": data["source_ref"],
+                "target_ref": data["target_ref"],
                 "allow_custom": True,
             }
             try:
@@ -488,7 +509,9 @@ def _build_stix_relationships(
 
 def _edge_type_to_stix(edge_type: str) -> str:
     """Map VoidAccess graph edge types to STIX relationship type strings."""
-    return RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
+    from export.relationships import _normalise_edge_type  # noqa: PLC0415
+    from graph.model import RELATIONSHIP_TYPE_STIX  # noqa: PLC0415
+    return RELATIONSHIP_TYPE_STIX.get(_normalise_edge_type(edge_type), "related-to")
 
 
 def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:

{voidaccess-1.9.1 → voidaccess-1.9.2}/graph/builder.py

@@ -789,8 +789,12 @@ def build_graph_from_db(
                     .join(
                         InvestigationEntityLink,
                         InvestigationEntityLink.entity_id == Entity.id,
+                        isouter=True,
+                    )
+                    .filter(
+                        (Entity.investigation_id == investigation_id)
+                        | (InvestigationEntityLink.investigation_id == investigation_id)
                     )
-                    .filter(InvestigationEntityLink.investigation_id == investigation_id)
                     .options(joinedload(Entity.page))
                 )
             else:
@@ -1230,8 +1234,12 @@ def build_graph_from_db_cached(investigation_id: uuid.UUID) -> nx.MultiDiGraph:
             .join(
                 InvestigationEntityLink,
                 InvestigationEntityLink.entity_id == Entity.id,
+                isouter=True,
+            )
+            .filter(
+                (Entity.investigation_id == investigation_id)
+                | (InvestigationEntityLink.investigation_id == investigation_id)
             )
-            .filter(InvestigationEntityLink.investigation_id == investigation_id)
             .options(joinedload(Entity.page))
             .yield_per(500)
         )

{voidaccess-1.9.1 → voidaccess-1.9.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.9.1"
+version = "1.9.2"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"

{voidaccess-1.9.1 → voidaccess-1.9.2}/vector/model_singleton.py

@@ -82,6 +82,11 @@ def get_embedding_model() -> "SentenceTransformer | None":
             logger.info("Loaded embedding model all-MiniLM-L6-v2 (singleton)")
         except Exception as exc:
             logger.warning("Failed to load embedding model: %s", exc)
+            logger.warning(
+                "Deterministic embedding fallback is active; install "
+                "voidaccess[nlp] (torch + sentence-transformers) for full "
+                "vector embeddings."
+            )
             _model = _FallbackEmbeddingModel()
 
     return _model

{voidaccess-1.9.1 → voidaccess-1.9.2}/voidaccess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.1
+Version: 1.9.2
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -172,6 +172,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
 
 Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
 
+### Optional vector embeddings
+
+The default installation can run without PyTorch. When the embedding stack is
+unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
+encoder. Install the optional NLP dependencies to enable full
+sentence-transformer vector embeddings:
+
+```bash
+pip install "voidaccess[nlp]"
+```
+
 ## LLM Providers
 
 | Provider | Typical models | Notes |

{voidaccess-1.9.1 → voidaccess-1.9.2}/voidaccess.egg-info/SOURCES.txt

@@ -62,6 +62,7 @@ export/__init__.py
 export/_entity_loading.py
 export/ioc_package.py
 export/misp.py
+export/relationships.py
 export/sigma.py
 export/snort_export.py
 export/stix.py

{voidaccess-1.9.1 → voidaccess-1.9.2}/voidaccess_cli/__init__.py

@@ -1,3 +1,3 @@
 """voidaccess CLI — dark-web OSINT command-line interface."""
 
-__version__ = "1.9.1"
+__version__ = "1.9.2"

{voidaccess-1.9.1 → voidaccess-1.9.2}/voidaccess_cli/commands/configure.py

@@ -418,7 +418,7 @@ def configure_proxy(
         api_label = "[green]enabled[/green]" if api_transport else "[red]disabled[/red]"
         proxy_label = "[green]enabled[/green]" if proxy_transport else "[red]disabled[/red]"
         console.print(f"  REST API transport      : {api_label}")
-        console.print(f"  Residential proxy       : {proxy_label}")
+        console.print(f"  Proxy transport         : {proxy_label}")
         console.print("  Scope           : paste + RSS only, never Tor")
         if proxy_transport and (not proxy_username or not proxy_password):
             console.print(
@@ -457,7 +457,7 @@ def configure_proxy(
         console.print(f"ScrapingAnt proxy transport [green]{state}[/green]")
         if enable_proxy and not cfg.get("enrichment_keys", {}).get("SCRAPINGANT_API_KEY"):
             console.print(
-                "[yellow]Note:[/yellow] verify that the separate proxy username/password pair is configured — "
+                "[yellow]Note:[/yellow] no SCRAPINGANT_API_KEY configured yet — "
                 "set one with `voidaccess configure proxy` (interactive) or "
                 "`voidaccess configure keys`."
             )
@@ -467,4 +467,3 @@ def configure_proxy(
     _prompt_scrapingant(cfg)
     cli_config.save_config(cfg)
     console.print(f"[green]Saved to[/green] {cli_config.CONFIG_PATH}")
-

{voidaccess-1.9.1 → voidaccess-1.9.2}/voidaccess_cli/commands/export.py

@@ -85,6 +85,13 @@ def run(
         target, suffix, fmt=fmt, query=(data.get("investigation") or {}).get("query")
     )
     out_path = Path(out_path).expanduser()
+    if out_path.exists():
+        safe_path = _collision_safe_output_path(out_path, fmt)
+        console.print(
+            f"[yellow]Warning:[/yellow] {out_path} already exists; "
+            f"writing {safe_path} to avoid overwriting it."
+        )
+        out_path = safe_path
     out_path.parent.mkdir(parents=True, exist_ok=True)
     if isinstance(payload, bytes):
         out_path.write_bytes(payload)
@@ -93,6 +100,16 @@ def run(
     console.print(f"[green]Wrote[/green] {out_path}")
 
 
+def _collision_safe_output_path(path: Path, fmt: str) -> Path:
+    """Return a format-aware unused path without overwriting an export."""
+    candidate = path.with_name(f"{path.stem}-{fmt}{path.suffix}")
+    counter = 2
+    while candidate.exists():
+        candidate = path.with_name(f"{path.stem}-{fmt}-{counter}{path.suffix}")
+        counter += 1
+    return candidate
+
+
 def _load_target(target: str) -> tuple[Optional[str], Optional[dict]]:
     p = Path(target).expanduser()
     if p.exists() and p.suffix == ".json":