voidaccess 1.9.2__tar.gz → 1.9.4__tar.gz

{voidaccess-1.9.2 → voidaccess-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.2
+Version: 1.9.4
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-1.9.2 → voidaccess-1.9.4}/api/main.py

@@ -40,7 +40,7 @@ logging.basicConfig(level=logging.DEBUG)
 # Rate limiter setup
 # ---------------------------------------------------------------------------
 
-DISABLE_RATE_LIMIT = os.getenv("DISABLE_RATE_LIMIT", "false").lower() == "true"
+DISABLE_RATE_LIMIT = (os.getenv("DISABLE_RATE_LIMIT", "false") or "false").lower() == "true"
 
 if DISABLE_RATE_LIMIT:
     limiter = None

{voidaccess-1.9.2 → voidaccess-1.9.4}/api/routes/entities.py

@@ -96,6 +96,7 @@ async def list_entities(
                         "context_snippet": e.context_snippet,
                         "context": e.context,
                         "investigation_id": str(e.investigation_id) if e.investigation_id else None,
+                        "investigation_count": getattr(e, "investigation_count", 1) or 1,
                         "created_at": e.created_at.isoformat() if e.created_at else None,
                     }
                     for e in entities
@@ -643,8 +644,9 @@ async def get_entity(
                 "freshness_label": freshness_display["label"],
                 "freshness_color": freshness_display["color"],
                 "source_count": entity.source_count or 1,
+                "investigation_count": getattr(entity, "investigation_count", 1) or 1,
                 "corroborating_sources": json.loads(entity.corroborating_sources or '["dark_web_scrape"]'),
-                "cross_referenced": (entity.source_count or 1) > 1,
+                "cross_referenced": (getattr(entity, "investigation_count", 1) or 1) > 1,
                 "defanged": defang,
                 "blockchain_data": {
                     "wallet_type": entity.entity_type if entity.entity_type in ("BITCOIN_ADDRESS", "ETHEREUM_ADDRESS", "MONERO_ADDRESS") else None,
@@ -811,6 +813,9 @@ def _entity_to_dict(entity) -> dict:  # type: ignore[type-arg]
         "investigation_id": str(entity.investigation_id) if entity.investigation_id else None,
         "created_at": entity.created_at.isoformat() if entity.created_at else None,
         "extraction_method": getattr(entity, "extraction_method", None),
+        "source_count": getattr(entity, "source_count", 1) or 1,
+        "investigation_count": getattr(entity, "investigation_count", 1) or 1,
+        "corroborating_sources": getattr(entity, "corroborating_sources", None),
     }
 
 

{voidaccess-1.9.2 → voidaccess-1.9.4}/api/routes/investigations.py

@@ -537,7 +537,7 @@ async def _check_cancelled(inv_uuid: uuid.UUID, investigation_id: str) -> bool:
 # Rate limiting (shared key_func with api/main.py; enforcement via app.state.limiter)
 # ---------------------------------------------------------------------------
 
-_DISABLE_RATE_LIMIT = os.getenv("DISABLE_RATE_LIMIT", "false").lower() == "true"
+_DISABLE_RATE_LIMIT = (os.getenv("DISABLE_RATE_LIMIT", "false") or "false").lower() == "true"
 
 if not _DISABLE_RATE_LIMIT:
     try:
@@ -2925,8 +2925,9 @@ async def get_investigation_entities(
                         "freshness_label": freshness_display["label"],
                         "freshness_color": freshness_display["color"],
                         "source_count": e.source_count or 1,
+                        "investigation_count": getattr(e, "investigation_count", 1) or 1,
                         "corroborating_sources": json.loads(e.corroborating_sources or '["dark_web_scrape"]'),
-                        "cross_referenced": (e.source_count or 1) > 1,
+                        "cross_referenced": (getattr(e, "investigation_count", 1) or 1) > 1,
                         "graph_node_id": graph_node_id,
                         "defanged": defang,
                     }

{voidaccess-1.9.2 → voidaccess-1.9.4}/db/migrations/env.py

@@ -11,7 +11,7 @@ import sys
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy import engine_from_config, pool
+from sqlalchemy import engine_from_config, inspect, pool, text
 
 # Make sure the project root is on sys.path so `from db.models import Base`
 # resolves correctly regardless of where alembic is invoked from.
@@ -36,6 +36,33 @@ if config.config_file_name is not None:
 target_metadata = Base.metadata
 
 
+def _ensure_version_column_capacity(connection) -> None:
+    """Keep legacy PostgreSQL Alembic tables able to store current revision IDs.
+
+    The initial Alembic table used VARCHAR(32), but later revision identifiers
+    are longer than 32 characters.  Widen the metadata column before Alembic
+    writes the next revision; this touches only migration bookkeeping, never
+    application data.
+    """
+    if connection.dialect.name != "postgresql":
+        return
+    inspector = inspect(connection)
+    if "alembic_version" not in inspector.get_table_names():
+        return
+    version_column = next(
+        (c for c in inspector.get_columns("alembic_version") if c["name"] == "version_num"),
+        None,
+    )
+    current_length = getattr(version_column.get("type"), "length", None) if version_column else None
+    if current_length is not None and current_length < 255:
+        connection.execute(
+            text(
+                "ALTER TABLE alembic_version "
+                "ALTER COLUMN version_num TYPE VARCHAR(255)"
+            )
+        )
+
+
 def run_migrations_offline() -> None:
     """
     Run migrations without a live DB connection.
@@ -64,6 +91,7 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,  # no pooling needed for one-shot migration runs
     )
     with connectable.connect() as connection:
+        _ensure_version_column_capacity(connection)
         context.configure(
             connection=connection,
             target_metadata=target_metadata,

voidaccess-1.9.4/db/migrations/versions/0026_add_entity_investigation_count.py

@@ -0,0 +1,46 @@
+"""Track distinct investigations independently from enrichment provenance."""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0026_add_entity_investigation_count"
+down_revision = "0025_add_cancellation_requested"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "entities",
+        sa.Column("investigation_count", sa.Integer(), server_default="1", nullable=False),
+    )
+
+    # Backfill from the existing cross-investigation links and include the
+    # owner column for rows created before links were consistently recorded.
+    conn = op.get_bind()
+    entity_rows = conn.execute(
+        sa.text("SELECT id, investigation_id FROM entities")
+    ).mappings().all()
+    link_rows = conn.execute(
+        sa.text("SELECT entity_id, investigation_id FROM investigation_entity_links")
+    ).mappings().all()
+    counts: dict[object, set[object]] = {}
+    for row in entity_rows:
+        counts[row["id"]] = (
+            {row["investigation_id"]}
+            if row["investigation_id"] is not None
+            else set()
+        )
+    for row in link_rows:
+        counts.setdefault(row["entity_id"], set()).add(row["investigation_id"])
+
+    for entity_id, investigation_ids in counts.items():
+        conn.execute(
+            sa.text("UPDATE entities SET investigation_count = :count WHERE id = :id"),
+            {"count": max(1, len(investigation_ids)), "id": entity_id},
+        )
+
+
+def downgrade() -> None:
+    op.drop_column("entities", "investigation_count")

{voidaccess-1.9.2 → voidaccess-1.9.4}/db/models.py

@@ -470,6 +470,11 @@ class Entity(Base):
     source_count: Mapped[int] = mapped_column(
         sa.Integer, server_default="1", default=1
     )
+    # Distinct investigations that independently surfaced this canonical
+    # entity. This is separate from enrichment-feed provenance below.
+    investigation_count: Mapped[int] = mapped_column(
+        sa.Integer, server_default="1", default=1
+    )
     corroborating_sources: Mapped[Optional[str]] = mapped_column(
         sa.Text, nullable=True
     )

{voidaccess-1.9.2 → voidaccess-1.9.4}/db/queries.py

@@ -281,7 +281,7 @@ def create_entity(
 
 def _link_entity_to_investigation(
     session: Session, entity_id: uuid.UUID, investigation_id: uuid.UUID
-) -> None:
+) -> bool:
     """Link an entity to an investigation via InvestigationEntityLink."""
     from db.models import InvestigationEntityLink
 
@@ -290,7 +290,7 @@ def _link_entity_to_investigation(
         entity_id=entity_id, investigation_id=investigation_id
     ).first()
     if existing:
-        return
+        return False
 
     # Also check pending (unflushed) objects — autoflush=False means they won't
     # appear in the query above, causing UNIQUE VIOLATION on batch flush.
@@ -300,13 +300,29 @@ def _link_entity_to_investigation(
             and obj.entity_id == entity_id
             and obj.investigation_id == investigation_id
         ):
-            return
+            return False
 
     link = InvestigationEntityLink(
         entity_id=entity_id,
         investigation_id=investigation_id
     )
     session.add(link)
+    return True
+
+
+def _entity_investigation_count(session: Session, entity: Entity) -> int:
+    """Count distinct investigations that surfaced *entity*, including owner."""
+    from db.models import InvestigationEntityLink
+
+    ids = {
+        row[0]
+        for row in session.query(InvestigationEntityLink.investigation_id)
+        .filter_by(entity_id=entity.id)
+        .all()
+    }
+    if entity.investigation_id:
+        ids.add(entity.investigation_id)
+    return max(1, len(ids))
 
 
 def upsert_entity_canonical(
@@ -359,9 +375,12 @@ def upsert_entity_canonical(
         existing.last_seen = datetime.now(timezone.utc)
         # Update last_seen_at for freshness tracking
         existing.last_seen_at = datetime.now(timezone.utc)
-        # Link to this investigation if not already linked
-        if existing.investigation_id != investigation_id:
-            _link_entity_to_investigation(session, existing.id, investigation_id)
+        # Link to this investigation if not already linked, including for
+        # legacy rows whose owner column predates the link table backfill.
+        linked = _link_entity_to_investigation(session, existing.id, investigation_id)
+        if linked:
+            session.flush()
+            existing.investigation_count = _entity_investigation_count(session, existing)
         return existing, False
     else:
         # Create new entity
@@ -380,6 +399,7 @@ def upsert_entity_canonical(
         session.add(entity)
         session.flush()  # populate entity.id before creating the link
         _link_entity_to_investigation(session, entity.id, investigation_id)
+        entity.investigation_count = 1
         return entity, True
 
 

{voidaccess-1.9.2 → voidaccess-1.9.4}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.9.2"
+PACKAGE_VERSION = "1.9.4"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.9.2 → voidaccess-1.9.4}/export/stix.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 import json
 import logging
 import os
+import re
 import warnings
 from typing import Any, Optional
 import uuid
@@ -78,6 +79,114 @@ _STIX_PATTERNS: dict[str, str] = {
 # Entity types that map to STIX Malware objects
 _MALWARE_TYPES = frozenset({"MALWARE_FAMILY", "RANSOMWARE_GROUP"})
 
+# STIX 2.1's malware-type vocabulary is intentionally small.  Keep the
+# mapping here instead of treating every MALWARE_FAMILY as the generic
+# ``malware`` type: family names and enrichment metadata often carry enough
+# information for SIEM consumers to filter useful categories.
+_STIX_MALWARE_TYPE_ALIASES = {
+    "infostealer": "spyware",
+    "stealer": "spyware",
+    "remote_access_trojan": "remote-access-trojan",
+    "remote access trojan": "remote-access-trojan",
+    "rat": "remote-access-trojan",
+    "coinminer": "resource-exploitation",
+    "cryptominer": "resource-exploitation",
+    "crypto-miner": "resource-exploitation",
+    "miner": "resource-exploitation",
+}
+_STIX_MALWARE_TYPES = frozenset({
+    "adware",
+    "backdoor",
+    "bot",
+    "ddos",
+    "dropper",
+    "exploit-kit",
+    "keylogger",
+    "ransomware",
+    "remote-access-trojan",
+    "resource-exploitation",
+    "rootkit",
+    "screen-capture",
+    "spyware",
+    "wiper",
+    "malware",
+})
+
+# These are conservative, high-signal family/name hints.  Unknown families
+# retain the valid generic STIX type rather than being assigned a speculative
+# category.
+_MALWARE_NAME_PATTERNS = (
+    ("ransomware", re.compile(
+        r"\b(?:ransomware|lockbit|blackcat|alphv|clop|conti|ryuk|hive|revil|darkside|wannacry)\b",
+        re.IGNORECASE,
+    )),
+    ("remote-access-trojan", re.compile(
+        r"\b(?:remote\s+access\s+trojan|remcos|njrat|nanocore|quasar|darkcomet|xrat)\b",
+        re.IGNORECASE,
+    )),
+    ("spyware", re.compile(
+        r"\b(?:spyware|infostealer|stealer|redline|vidar|lumma|raccoon|agent\s+tesla|formbook)\b",
+        re.IGNORECASE,
+    )),
+    ("bot", re.compile(
+        r"\b(?:botnet|emotet|qakbot|qbot|trickbot|mirai|zeus|mozi)\b",
+        re.IGNORECASE,
+    )),
+    ("keylogger", re.compile(r"\bkeylogger\b", re.IGNORECASE)),
+    ("exploit-kit", re.compile(r"\bexploit[\s-]+kit\b", re.IGNORECASE)),
+    ("rootkit", re.compile(r"\brootkit\b", re.IGNORECASE)),
+    ("screen-capture", re.compile(r"\bscreen[\s-]+capture\b", re.IGNORECASE)),
+    ("dropper", re.compile(r"\bdropper\b", re.IGNORECASE)),
+    ("wiper", re.compile(r"\bwiper\b", re.IGNORECASE)),
+    ("adware", re.compile(r"\badware\b", re.IGNORECASE)),
+    ("resource-exploitation", re.compile(
+        r"\b(?:cryptominer|crypto[\s-]+miner|coinminer|cryptojacking)\b",
+        re.IGNORECASE,
+    )),
+    ("backdoor", re.compile(r"\bbackdoor\b", re.IGNORECASE)),
+)
+
+
+def _entity_attribute(entity: Any, name: str) -> Any:
+    """Read optional classification metadata from dicts and ORM-like rows."""
+    if isinstance(entity, dict):
+        return entity.get(name)
+    return getattr(entity, name, None)
+
+
+def _normalise_malware_type(value: Any) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    key = value.strip().lower().replace("_", " ")
+    key = _STIX_MALWARE_TYPE_ALIASES.get(key, key)
+    key = key.replace(" ", "-")
+    return key if key in _STIX_MALWARE_TYPES else None
+
+
+def _classify_malware(entity: Any) -> list[str]:
+    """Return evidence-backed STIX malware types for an entity."""
+    entity_type = str(getattr(entity, "entity_type", "") or "").upper()
+    if isinstance(entity, dict):
+        entity_type = str(entity.get("entity_type") or "").upper()
+    if entity_type == "RANSOMWARE_GROUP":
+        return ["ransomware"]
+
+    for field_name in ("malware_types", "malware_type", "malware_category", "category", "family_type"):
+        raw = _entity_attribute(entity, field_name)
+        values = raw if isinstance(raw, (list, tuple, set)) else [raw]
+        classified = [item for item in (_normalise_malware_type(v) for v in values) if item]
+        if classified:
+            return list(dict.fromkeys(classified))
+
+    evidence = " ".join(
+        str(_entity_attribute(entity, field) or "")
+        for field in ("value", "context_snippet")
+    )
+    for malware_type, pattern in _MALWARE_NAME_PATTERNS:
+        if pattern.search(evidence):
+            return [malware_type]
+    return ["malware"]
+
 # ---------------------------------------------------------------------------
 # Confidence mapping: VoidAccess float → STIX integer (0-100)
 # ---------------------------------------------------------------------------
@@ -160,11 +269,7 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
         malware = stix2.Malware(
             name=entity.value,
             is_family=True,
-            malware_types=(
-                ["ransomware"]
-                if entity.entity_type == "RANSOMWARE_GROUP"
-                else ["malware"]
-            ),
+            malware_types=_classify_malware(entity),
             confidence=_to_stix_confidence(get_entity_confidence(entity)),
             allow_custom=True,
             x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),

{voidaccess-1.9.2 → voidaccess-1.9.4}/extractor/entity_shape.py

@@ -93,14 +93,7 @@ def _load_words() -> None:
 
 
 # Organisation suffixes that strongly signal a real org name.
-_ORG_SUFFIXES: frozenset[str] = frozenset({
-    "inc", "incorporated", "llc", "ltd", "limited", "corp", "corporation",
-    "gmbh", "ag", "sa", "srl", "bv", "plc", "co", "company", "group",
-    "holdings", "technologies", "systems", "solutions", "lab", "labs", "laboratories",
-    "networks", "software", "security", "capital", "partners", "ventures",
-    "bank", "university", "institute", "foundation", "agency", "bureau",
-    "industries", "international", "global", "media", "digital", "consulting",
-})
+_ORG_SUFFIXES = gazetteer.ORG_SUFFIXES
 
 _TOKEN_RE = re.compile(r"[A-Za-z0-9]+(?:['’][A-Za-z]+)?")
 _WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")

{voidaccess-1.9.2 → voidaccess-1.9.4}/extractor/gazetteer.py

@@ -42,6 +42,24 @@ _DATA_PATH = os.path.join(
     "data",
     "threat_gazetteer.json",
 )
+_DOMAIN_STOPWORDS_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "data",
+    "domain_stopwords_en.txt",
+)
+
+# Organisation suffixes are also generic name qualifiers when they appear on
+# an actor/malware candidate (for example, ``LockBit Ransomware Group``).
+# Keep this set here so gazetteer matching and shape validation use the same
+# vocabulary without introducing a circular import through entity_shape.
+ORG_SUFFIXES: frozenset[str] = frozenset({
+    "inc", "incorporated", "llc", "ltd", "limited", "corp", "corporation",
+    "gmbh", "ag", "sa", "srl", "bv", "plc", "co", "company", "group",
+    "holdings", "technologies", "systems", "solutions", "lab", "labs", "laboratories",
+    "networks", "software", "security", "capital", "partners", "ventures",
+    "bank", "university", "institute", "foundation", "agency", "bureau",
+    "industries", "international", "global", "media", "digital", "consulting",
+})
 
 # Minimum canonical-key length — drops junk snapshot entries like "$$$" (which
 # canonicalise to "") or single stray characters that would over-match.
@@ -52,6 +70,7 @@ _actors: frozenset[str] = frozenset()
 _malware: frozenset[str] = frozenset()
 _ransomware: frozenset[str] = frozenset()
 _generated_at: Optional[str] = None
+_generic_terms: frozenset[str] = frozenset()
 
 
 def _canonical_key(name: str) -> str:
@@ -77,8 +96,38 @@ def _keyset(names) -> frozenset[str]:
     return frozenset(out)
 
 
+def _load_generic_terms() -> frozenset[str]:
+    """Load generic edge qualifiers used for suffix/prefix retry matching."""
+    terms = set(ORG_SUFFIXES)
+    try:
+        with open(_DOMAIN_STOPWORDS_PATH, encoding="utf-8") as fh:
+            terms.update(line.strip().lower() for line in fh if line.strip())
+    except FileNotFoundError:
+        logger.warning("Domain stopword lexicon not found at %s", _DOMAIN_STOPWORDS_PATH)
+    return frozenset(terms)
+
+
+def _lookup_keys(name: str) -> tuple[str, ...]:
+    """Return the original key and edge-stripped retry keys for *name*."""
+    original = _canonical_key(name)
+    if not name:
+        return (original,)
+
+    terms = _generic_terms
+    tokens = re.findall(r"[A-Za-z0-9]+", name.lower())
+    while tokens and tokens[0] in terms:
+        tokens.pop(0)
+    while tokens and tokens[-1] in terms:
+        tokens.pop()
+
+    stripped = _canonical_key(" ".join(tokens))
+    if stripped and stripped != original:
+        return (original, stripped)
+    return (original,)
+
+
 def _load() -> None:
-    global _loaded, _actors, _malware, _ransomware, _generated_at
+    global _loaded, _actors, _malware, _ransomware, _generated_at, _generic_terms
     if _loaded:
         return
     _loaded = True
@@ -89,6 +138,7 @@ def _load() -> None:
         _malware = _keyset(data.get("malware"))
         _ransomware = _keyset(data.get("ransomware"))
         _generated_at = data.get("generated_at")
+        _generic_terms = _load_generic_terms()
         logger.info(
             "Gazetteer loaded: %d actors, %d malware, %d ransomware (generated %s)",
             len(_actors), len(_malware), len(_ransomware), _generated_at,
@@ -106,17 +156,17 @@ def _load() -> None:
 
 def is_known_actor(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _actors
+    return any(key in _actors for key in _lookup_keys(name))
 
 
 def is_known_malware(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _malware
+    return any(key in _malware for key in _lookup_keys(name))
 
 
 def is_known_ransomware(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _ransomware
+    return any(key in _ransomware for key in _lookup_keys(name))
 
 
 def is_known(name: str, category: str) -> bool:
@@ -132,12 +182,12 @@ def is_known(name: str, category: str) -> bool:
 def category_of(name: str) -> Optional[str]:
     """Return the most specific known category for *name*, or None."""
     _load()
-    key = _canonical_key(name)
-    if key in _ransomware:
+    keys = _lookup_keys(name)
+    if any(key in _ransomware for key in keys):
         return "ransomware"
-    if key in _malware:
+    if any(key in _malware for key in keys):
         return "malware"
-    if key in _actors:
+    if any(key in _actors for key in keys):
         return "threat_actor"
     return None
 

{voidaccess-1.9.2 → voidaccess-1.9.4}/extractor/ner.py

@@ -20,6 +20,7 @@ import logging
 import re
 
 from extractor import entity_shape as _shape
+from extractor import gazetteer
 
 logger = logging.getLogger(__name__)
 
@@ -493,12 +494,31 @@ def extract_named_entities(text: str) -> dict[str, list[str]]:
                 for ent in doc.ents:
                     if ent.label_ == "ORG":
                         candidate = ent.text.strip()
+                        # A known actor can be returned by spaCy as an ORG when
+                        # the surrounding generic qualifier is included in the
+                        # span (for example, "LockBit Ransomware Group").
+                        # Preserve the extracted value but emit the gazetteer
+                        # classification so downstream normalization does not
+                        # misfile it as ORGANIZATION_NAME.
+                        category = gazetteer.category_of(candidate)
+                        if category == "threat_actor":
+                            result[THREAT_ACTOR_HANDLE].append(candidate)
+                            continue
+                        if category == "ransomware":
+                            result[RANSOMWARE_GROUP].append(candidate)
+                            continue
+                        if category == "malware":
+                            result[MALWARE_FAMILY].append(candidate)
+                            continue
                         # Accept only if it has organisation shape (proper-noun
                         # structure, org suffix, brand casing) — not merely
                         # "absent from a denylist".
                         if _shape.evaluate(ORGANIZATION_NAME, candidate).accept:
                             orgs.append(candidate)
                 result[ORGANIZATION_NAME] = _dedup(orgs)
+                result[THREAT_ACTOR_HANDLE] = _dedup(result[THREAT_ACTOR_HANDLE])
+                result[MALWARE_FAMILY] = _dedup(result[MALWARE_FAMILY])
+                result[RANSOMWARE_GROUP] = _dedup(result[RANSOMWARE_GROUP])
 
     except Exception:
         logger.exception("extract_named_entities encountered an unexpected error")

{voidaccess-1.9.2 → voidaccess-1.9.4}/extractor/normalizer.py

@@ -30,6 +30,7 @@ from urllib.parse import urlparse
 
 from extractor import entity_shape as _shape
 from extractor import confidence as _conf
+from extractor import gazetteer as _gazetteer
 
 logger = logging.getLogger(__name__)
 
@@ -1463,11 +1464,25 @@ def normalize_entities(
 
     noise_count = 0
 
-    for entity_type, values in raw_entities.items():
+    for raw_entity_type, values in raw_entities.items():
         for raw_value in values:
             if not raw_value or not raw_value.strip():
                 continue
 
+            entity_type = raw_entity_type
+
+            # NER may label a known actor/malware span as ORGANIZATION_NAME
+            # when generic qualifiers are included in the span.  Let the
+            # gazetteer classification win after its suffix/prefix retry,
+            # while leaving unrelated organization candidates unchanged.
+            if entity_type == "ORGANIZATION_NAME":
+                known_category = _gazetteer.category_of(raw_value)
+                entity_type = {
+                    "threat_actor": "THREAT_ACTOR_HANDLE",
+                    "malware": "MALWARE_FAMILY",
+                    "ransomware": "RANSOMWARE_GROUP",
+                }.get(known_category, entity_type)
+
             if entity_type in ("FILE_HASH_MD5", "FILE_HASH_SHA1", "FILE_HASH_SHA256"):
                 if not _validate_hash_length(entity_type, raw_value):
                     logger.debug(

{voidaccess-1.9.2 → voidaccess-1.9.4}/graph/builder.py

@@ -39,6 +39,22 @@ logger = logging.getLogger(__name__)
 
 _CO_OCCURRENCE_MAX_ENTITIES_PER_PAGE = 40
 _CO_OCCURRENCE_MAX_EDGES_PER_PAGE = 120
+
+# ORGANIZATION_NAME is deliberately absent from both sets below, even though it
+# is the largest (or second-largest) bucket in most real runs.  These sets were
+# introduced by v1.7.2, the same release that filtered "ORGANIZATION_NAME NER
+# noise ... generic security/programming vocabulary false positives" — the
+# exclusion is that fix, not an oversight.  The bucket is still ~25-45% junk on
+# measured runs (values like POST, JavaScript, HTML, HTTPS, IP, SCADA, EDR sit
+# alongside genuine orgs), so admitting it here would attach hundreds of
+# meaningless edges to every real hub and drown the signal the semantic filter
+# exists to preserve.
+#
+# Dependency, not a TODO: genuinely-real organizations *should* eventually be
+# able to participate in co-occurrence edges.  That is gated on Phase 2's
+# noise-reduction work actually improving what lands in this bucket — do not
+# widen eligibility before then, and re-measure the bucket's noise rate rather
+# than assuming Phase 2 fixed it.
 _CO_OCCURRENCE_HUB_TYPES = frozenset({
     "THREAT_ACTOR_HANDLE",
     "RANSOMWARE_GROUP",

{voidaccess-1.9.2 → voidaccess-1.9.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.9.2"
+version = "1.9.4"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"