voidaccess 1.8.1__tar.gz → 1.9.0__tar.gz

{voidaccess-1.8.1/voidaccess.egg-info → voidaccess-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.8.1
+Version: 1.9.0
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-1.8.1 → voidaccess-1.9.0}/api/routes/entities.py

@@ -22,6 +22,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
 from fastapi.responses import Response
 from api.auth import CurrentUser, get_current_user
 from api.errors import GENERIC_ERROR_MESSAGE, log_exception
+from extractor.identity import entity_canonical_id
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -321,7 +322,9 @@ async def get_stylometry_analysis(
             if entity is None:
                 raise HTTPException(status_code=404, detail="Entity not found")
 
-            canonical = entity.canonical_value or entity.value.lower()
+            # Canonical dedup key via the shared identity module (was an inline
+            # `entity.canonical_value or entity.value.lower()` re-derivation).
+            canonical = entity_canonical_id(entity)
 
             related = (
                 session.query(Entity)
@@ -406,7 +409,7 @@ async def get_stylometry_analysis(
                     similar_actors = await asyncio.to_thread(
                         _find_similar_actors,
                         profile=profile,
-                        canonical_value=entity.canonical_value or entity.value.lower(),
+                        canonical_value=entity_canonical_id(entity),
                         entity_type=entity.entity_type,
                     )
             except Exception as e:
@@ -474,7 +477,9 @@ async def get_opsec_analysis(
             if entity is None:
                 raise HTTPException(status_code=404, detail="Entity not found")
 
-            canonical = entity.canonical_value or entity.value.lower()
+            # Canonical dedup key via the shared identity module (was an inline
+            # `entity.canonical_value or entity.value.lower()` re-derivation).
+            canonical = entity_canonical_id(entity)
             related = (
                 session.query(Entity)
                 .filter(
@@ -827,17 +832,44 @@ def _get_entity_value(entity_id: str) -> Optional[str]:
         return None
 
 
-def _resolve_graph_node_id(graph, entity_value: str) -> Optional[str]:
-    """Resolve graph node by exact value, then by handle@domain prefix."""
+def _resolve_graph_node_id(
+    graph, entity_value: str, entity_type: Optional[str] = None
+) -> Optional[str]:
+    """Resolve a graph node id for an entity value.
+
+    The graph is keyed by canonical identity (extractor.identity.entity_graph_id).
+    When ``entity_type`` is known, the lookup value is canonicalised through the
+    same shared module so it matches the node regardless of the casing the value
+    was observed in — closing the case-sensitivity gap the audit flagged.
+    Without a type it falls back to the historical exact-then-``handle@domain``
+    prefix behaviour.
+    """
     if graph is None:
         return None
-    if graph.has_node(entity_value):
-        return entity_value
 
-    prefix = f"{entity_value}@"
-    for node_id in graph.nodes:
-        if isinstance(node_id, str) and node_id.startswith(prefix):
-            return node_id
+    candidates = [entity_value]
+    if entity_type:
+        try:
+            from types import SimpleNamespace  # noqa: PLC0415
+
+            canon = entity_canonical_id(
+                SimpleNamespace(entity_type=entity_type, value=entity_value)
+            )
+            if canon and canon not in candidates:
+                candidates.append(canon)
+        except Exception:
+            pass
+
+    # 1) exact match on any candidate
+    for cand in candidates:
+        if graph.has_node(cand):
+            return cand
+    # 2) handle@domain prefix match on any candidate
+    for cand in candidates:
+        prefix = f"{cand}@"
+        for node_id in graph.nodes:
+            if isinstance(node_id, str) and node_id.startswith(prefix):
+                return node_id
     return None
 
 

{voidaccess-1.8.1 → voidaccess-1.9.0}/db/queries.py

@@ -348,8 +348,8 @@ def upsert_entity_canonical(
     
     if existing:
         # Update confidence if new extraction is more confident
-        if confidence > existing.confidence:
-            existing.confidence = confidence
+        from extractor.confidence import get_entity_confidence
+        existing.confidence = get_entity_confidence(existing, confidence)
         # Update context if we have a better snippet
         if context_snippet and len(context_snippet) > len(existing.context_snippet or ""):
             existing.context_snippet = context_snippet

voidaccess-1.9.0/export/_entity_loading.py

@@ -0,0 +1,46 @@
+"""
+export/_entity_loading.py — one shared DB-row → NormalizedEntity mapping for
+every exporter.
+
+Previously STIX, MISP, and Sigma each reconstructed a ``NormalizedEntity`` from
+a DB ``Entity`` row independently, and populated *different* fields — STIX set
+``source_quality`` and ``extraction_method``, MISP and Sigma did not.  A future
+consumer reading ``entity.source_quality`` therefore got different behaviour
+depending on which export path produced the entity (audit finding: STIX/MISP
+field divergence).  Centralising the construction here closes that drift: all
+three exporters now produce identically-populated entities.
+
+The value mapping is preserved exactly as it was in all three call sites
+(``canonical_value or value``) so export outputs are unchanged; only the
+previously-inconsistent auxiliary fields are now uniform.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from extractor.normalizer import NormalizedEntity
+
+
+def normalized_entity_from_db_row(e: Any) -> NormalizedEntity:
+    """Build a NormalizedEntity from a DB Entity row with consistent fields.
+
+    Used by every exporter (STIX / MISP / Sigma) so field population can never
+    drift between export paths again.
+    """
+    source_url = ""
+    try:
+        if e.page:
+            source_url = e.page.url or ""
+    except Exception:
+        pass
+    return NormalizedEntity(
+        entity_type=e.entity_type,
+        value=e.canonical_value or e.value,
+        confidence=e.confidence,
+        source_url=source_url,
+        page_id=e.page_id,
+        context_snippet=e.context_snippet or "",
+        extraction_method="db",
+        source_quality=getattr(e, "source_quality", 1.0),
+    )

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/ioc_package.py

@@ -45,8 +45,11 @@ import logging
 import re
 import zipfile
 from datetime import datetime, timezone
+from types import SimpleNamespace
 from typing import Any, Iterable, Optional
 
+from extractor.identity import entity_canonical_id
+
 logger = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
@@ -54,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.8.1"
+PACKAGE_VERSION = "1.9.0"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.
@@ -138,19 +141,28 @@ def _entity_type(e: Any) -> str:
 
 
 def _entity_value(e: Any) -> str:
-    """Return the entity's primary value, preferring canonical_value."""
+    """Return the entity's canonical value via the shared identity module.
+
+    Migrated (audit finding 1.3 / 3.9 item 2) from the old
+    ``entity.canonical_value or entity.value`` pattern to call
+    ``extractor.identity.entity_canonical_id`` directly.  Phase 1 already fixed
+    ``NormalizedEntity.canonical_value`` so that old pattern no longer returned
+    the raw value for pipeline-stage entities — but this makes the choice
+    explicit at the call site (one shared function, not an assumption about
+    what a property returns), so DB-backed and pipeline-stage versions of the
+    same entity can never produce inconsistent casing here again.
+    """
+    etype = _entity_type(e)
     if isinstance(e, dict):
-        return (
-            e.get("canonical_value")
-            or e.get("value")
-            or e.get("canonical")
-            or ""
-        ) or ""
-    return (
-        getattr(e, "canonical_value", None)
-        or getattr(e, "value", None)
-        or ""
-    ) or ""
+        raw = e.get("value") or e.get("canonical_value") or e.get("canonical") or ""
+    else:
+        raw = getattr(e, "value", None) or getattr(e, "canonical_value", None) or ""
+    if not raw:
+        return ""
+    try:
+        return entity_canonical_id(SimpleNamespace(entity_type=etype, value=raw))
+    except Exception:
+        return raw
 
 
 def _entity_confidence(e: Any) -> float:
@@ -730,9 +742,7 @@ class _DictEntityAdapter:
 
     @property
     def canonical_value(self) -> str:
-        if isinstance(self._d, dict):
-            return self._d.get("canonical_value") or self._d.get("value") or ""
-        return ""
+        return entity_canonical_id(self._d)
 
     @property
     def confidence(self) -> float:

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/misp.py

@@ -209,23 +209,14 @@ def _load_investigation_and_entities(
                 want = frozenset(filter_uuids)
                 db_entities = [e for e in db_entities if e.id in want]
 
-            normalized: list[NormalizedEntity] = []
-            for e in db_entities:
-                source_url = ""
-                try:
-                    if e.page:
-                        source_url = e.page.url or ""
-                except Exception:
-                    pass
-                ne = NormalizedEntity(
-                    entity_type=e.entity_type,
-                    value=e.canonical_value or e.value,
-                    confidence=e.confidence,
-                    source_url=source_url,
-                    page_id=e.page_id,
-                    context_snippet=e.context_snippet or "",
-                )
-                normalized.append(ne)
+            from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+
+            # Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
+            # auxiliary fields (source_quality, extraction_method) no longer
+            # differ between export paths.
+            normalized: list[NormalizedEntity] = [
+                normalized_entity_from_db_row(e) for e in db_entities
+            ]
 
             session.expunge_all()
             return investigation, normalized

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/sigma.py

@@ -307,22 +307,12 @@ def _load_entities_for_investigation(investigation_id: Any) -> list[Any]:
                 .all()
             )
 
-            result: list[NormalizedEntity] = []
-            for e in db_entities:
-                source_url = ""
-                try:
-                    if e.page:
-                        source_url = e.page.url or ""
-                except Exception:
-                    pass
-                result.append(NormalizedEntity(
-                    entity_type=e.entity_type,
-                    value=e.canonical_value or e.value,
-                    confidence=e.confidence,
-                    source_url=source_url,
-                    page_id=e.page_id,
-                    context_snippet=e.context_snippet or "",
-                ))
+            from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
+
+            # Shared row→NormalizedEntity mapping (same one STIX/MISP use).
+            result: list[NormalizedEntity] = [
+                normalized_entity_from_db_row(e) for e in db_entities
+            ]
             return result
 
     except Exception as exc:

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/snort_export.py

@@ -44,6 +44,8 @@ from datetime import datetime, timezone
 from typing import Any
 from urllib.parse import urlparse
 
+from extractor.identity import entity_canonical_id
+
 # ---------------------------------------------------------------------------
 # Constants
 # ---------------------------------------------------------------------------
@@ -104,18 +106,7 @@ def _entity_type(e: Any) -> str:
 
 
 def _entity_value(e: Any) -> str:
-    if isinstance(e, dict):
-        return (
-            e.get("canonical_value")
-            or e.get("value")
-            or e.get("canonical")
-            or ""
-        ) or ""
-    return (
-        getattr(e, "canonical_value", None)
-        or getattr(e, "value", None)
-        or ""
-    ) or ""
+    return entity_canonical_id(e)
 
 
 def _entity_confidence(e: Any) -> float:

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/stix.py

@@ -22,6 +22,9 @@ import warnings
 from typing import Any, Optional
 import uuid
 
+from graph.model import RELATIONSHIP_TYPE_STIX
+from extractor.confidence import get_entity_confidence
+
 logger = logging.getLogger(__name__)
 
 # MED-9: suppress SQLAlchemy SAWarning from ORM IN-clause coercion internals.
@@ -127,7 +130,7 @@ def entity_to_stix_indicator(entity: Any) -> Optional[Any]:
             pattern=pattern,
             pattern_type="stix",
             indicator_types=indicator_types,
-            confidence=_to_stix_confidence(entity.confidence),
+            confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
             external_references=(
@@ -158,7 +161,7 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         malware = stix2.Malware(
             name=entity.value,
             is_family=True,
-            confidence=_to_stix_confidence(entity.confidence),
+            confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
             external_references=(
@@ -189,7 +192,7 @@ def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
         threat_actor = stix2.ThreatActor(
             name=entity.value,
             aliases=[entity.value],
-            confidence=_to_stix_confidence(entity.confidence),
+            confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
             external_references=(
@@ -241,30 +244,28 @@ def investigation_to_stix_bundle(
     if not entities:
         return _empty_bundle()
 
-    try:
-        from graph.builder import _make_node_id  # noqa: PLC0415
-    except Exception:  # pragma: no cover — graph layer optional
-        def _make_node_id(entity_type, value, source_url):  # type: ignore
-            return value
+    from extractor.identity import entity_canonical_id, entity_graph_id  # noqa: PLC0415
 
     stix_objects: list[Any] = []
-    # Maps BOTH the raw entity.value AND the graph node_id → stix_object.id.
-    # Graph edges are keyed by node_id (which for THREAT_ACTOR_HANDLE is
-    # "value@domain", not the bare value), so registering only entity.value
-    # here silently dropped every threat-actor relationship — the exact
-    # "silently dropped relationships" failure mode the CHANGELOG keeps
-    # re-fixing.  Registering the node_id alias lets typed edges (which very
-    # often have a threat-actor endpoint) survive into the bundle.
+    # Maps BOTH the graph node id AND the bare canonical value → stix_object.id.
+    # Graph edges are keyed by the graph node id (which for THREAT_ACTOR_HANDLE
+    # is "<canonical>@domain", not the bare value); registering only the bare
+    # value silently dropped every threat-actor relationship — the exact
+    # "silently dropped relationships" failure the CHANGELOG keeps re-fixing.
+    #
+    # Both keys are now derived through extractor.identity, the *same* module
+    # the graph builder uses to generate node ids.  The two sides can no longer
+    # diverge (e.g. "LockBit" vs "lockbit") because they are literally the same
+    # function call, not two functions that happen to agree today.
     stix_id_map: dict[str, str] = {}
 
     def _register(entity: Any, stix_id: str) -> None:
-        stix_id_map.setdefault(entity.value, stix_id)
         try:
-            node_id = _make_node_id(
-                entity.entity_type,
-                entity.value,
-                getattr(entity, "source_url", "") or "",
-            )
+            stix_id_map.setdefault(entity_canonical_id(entity), stix_id)
+        except Exception:
+            stix_id_map.setdefault(entity.value, stix_id)
+        try:
+            node_id = entity_graph_id(entity)
         except Exception:
             node_id = entity.value
         if node_id:
@@ -381,6 +382,7 @@ def _load_entities_for_investigation(
         from db.queries import get_investigation_by_id_or_run  # noqa: PLC0415
         from db.models import Entity, InvestigationEntityLink  # noqa: PLC0415
         from extractor.normalizer import NormalizedEntity  # noqa: PLC0415
+        from export._entity_loading import normalized_entity_from_db_row  # noqa: PLC0415
 
         inv_uuid = _coerce_uuid(investigation_id)
         if inv_uuid is None:
@@ -407,24 +409,9 @@ def _load_entities_for_investigation(
                 want = frozenset(entity_ids)
                 db_entities = [e for e in db_entities if e.id in want]
 
-            result: list[NormalizedEntity] = []
-            for e in db_entities:
-                source_url = ""
-                try:
-                    if e.page:
-                        source_url = e.page.url or ""
-                except Exception:
-                    pass
-                result.append(NormalizedEntity(
-                    entity_type=e.entity_type,
-                    value=e.canonical_value or e.value,
-                    confidence=e.confidence,
-                    source_url=source_url,
-                    page_id=e.page_id,
-                    context_snippet=e.context_snippet or "",
-                    extraction_method="db",
-                    source_quality=getattr(e, "source_quality", 1.0),
-                ))
+            result: list[NormalizedEntity] = [
+                normalized_entity_from_db_row(e) for e in db_entities
+            ]
             return result
 
         if session is not None:
@@ -501,29 +488,7 @@ def _build_stix_relationships(
 
 def _edge_type_to_stix(edge_type: str) -> str:
     """Map VoidAccess graph edge types to STIX relationship type strings."""
-    mapping = {
-        "CO_APPEARED_ON": "related-to",
-        "CO_INVESTIGATION": "related-to",
-        "POSTED_BY": "attributed-to",
-        "LINKED_TO": "related-to",
-        "PAID_TO": "related-to",
-        "MEMBER_OF": "member-of",
-        "USES": "uses",
-        "CLAIMED": "attributed-to",
-        "LIKELY_SAME_ACTOR": "related-to",
-        "CONFIRMED_SAME_ACTOR": "related-to",
-        "FUNDED_BY": "related-to",
-        # Typed relationships from the LLM relationship-extraction pass.  These
-        # map to documented STIX 2.1 common relationship types where one
-        # exists; CONTROLS has no standard STIX verb so it degrades to the
-        # safe "related-to" default rather than emitting a non-spec type.
-        "DROPS": "drops",
-        "TARGETS": "targets",
-        "EXPLOITS": "exploits",
-        "COMMUNICATES_WITH": "communicates-with",
-        "CONTROLS": "related-to",
-    }
-    return mapping.get(edge_type, "related-to")
+    return RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
 
 
 def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:

{voidaccess-1.8.1 → voidaccess-1.9.0}/export/yara_export.py

@@ -35,6 +35,8 @@ import re
 from datetime import datetime, timezone
 from typing import Any, Optional
 
+from extractor.identity import entity_canonical_id
+
 # ---------------------------------------------------------------------------
 # Constants
 # ---------------------------------------------------------------------------
@@ -115,19 +117,7 @@ def _entity_type(e: Any) -> str:
 
 
 def _entity_value(e: Any) -> str:
-    """Return the entity's primary value, preferring canonical_value."""
-    if isinstance(e, dict):
-        return (
-            e.get("canonical_value")
-            or e.get("value")
-            or e.get("canonical")
-            or ""
-        ) or ""
-    return (
-        getattr(e, "canonical_value", None)
-        or getattr(e, "value", None)
-        or ""
-    ) or ""
+    return entity_canonical_id(e)
 
 
 def _entity_confidence(e: Any) -> float:

{voidaccess-1.8.1 → voidaccess-1.9.0}/extractor/confidence.py

@@ -26,6 +26,10 @@ in later, at the batch cap stage, because it isn't known per-page — see
 
 The result is a genuinely differentiated value per extraction, not a category.
 Every function is total and never raises.
+
+Confidence is monotonic after extraction: new evidence may raise an entity's
+stored confidence, but never lowers it.  ``get_entity_confidence`` is the
+single read/merge helper for persisted or in-memory entity objects.
 """
 
 from __future__ import annotations
@@ -125,6 +129,33 @@ def compute_confidence(
         return 0.6
 
 
+def get_entity_confidence(entity, candidate: float | None = None) -> float:
+    """Return the canonical confidence, optionally merging new evidence.
+
+    The monotonic max rule is shared by extraction batches, database upserts,
+    and downstream consumers.  Values are always clamped to ``[0.0, 1.0]``.
+    """
+    try:
+        current = float(getattr(entity, "confidence", entity) or 0.0)
+        current = max(0.0, min(1.0, current))
+        if candidate is None:
+            return current
+        return max(current, max(0.0, min(1.0, float(candidate))))
+    except (TypeError, ValueError, AttributeError):
+        return 0.0
+
+
+def apply_evidence(entity, *signals: float) -> float:
+    """Raise an entity confidence from additive evidence without compounding."""
+    try:
+        evidence = sum(max(0.0, float(signal)) for signal in signals)
+    except (TypeError, ValueError):
+        evidence = 0.0
+    value = min(1.0, get_entity_confidence(entity) + evidence)
+    entity.confidence = get_entity_confidence(entity, value)
+    return entity.confidence
+
+
 def corroboration_boost(distinct_sources: int) -> float:
     """Confidence lift from independent corroboration.