voidaccess 1.7.1__tar.gz → 1.7.2__tar.gz

{voidaccess-1.7.1 → voidaccess-1.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.7.1
+Version: 1.7.2
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -104,11 +104,12 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.7.1
-- STIX export now produces real bundles with entities and relationships again.
-- Entity extraction on CVE/financial queries now returns the expected entity mix instead of collapsing to organization-name noise.
-- Confidence scores now show multi-value spread that reflects source quality, extraction method, and corroboration.
-- Entity readback now includes rows linked through `InvestigationEntityLink` and dedupes canonical collisions correctly.
+## What's New in v1.7.2
+- STIX export relationship generation is bounded and semantic, avoiding page-level cartesian edge explosions at larger entity counts.
+- Repeated NER false positives for threat actor handles and organization names are filtered before persistence.
+- MISP/Sigma/API query paths no longer use the SQLAlchemy subquery-in-`IN()` pattern that produced warnings.
+- Optional configuration warnings are summarized once per process.
+- Refined LLM query output is persisted as a short search string, not explanatory Markdown.
 
 ---
 
@@ -585,4 +586,3 @@ To report a security vulnerability, see [docs/SECURITY.md](docs/SECURITY.md).
 
 MIT License. See [LICENSE](LICENSE) for details.
 
-

{voidaccess-1.7.1 → voidaccess-1.7.2}/README.md

@@ -19,11 +19,12 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.7.1
-- STIX export now produces real bundles with entities and relationships again.
-- Entity extraction on CVE/financial queries now returns the expected entity mix instead of collapsing to organization-name noise.
-- Confidence scores now show multi-value spread that reflects source quality, extraction method, and corroboration.
-- Entity readback now includes rows linked through `InvestigationEntityLink` and dedupes canonical collisions correctly.
+## What's New in v1.7.2
+- STIX export relationship generation is bounded and semantic, avoiding page-level cartesian edge explosions at larger entity counts.
+- Repeated NER false positives for threat actor handles and organization names are filtered before persistence.
+- MISP/Sigma/API query paths no longer use the SQLAlchemy subquery-in-`IN()` pattern that produced warnings.
+- Optional configuration warnings are summarized once per process.
+- Refined LLM query output is persisted as a short search string, not explanatory Markdown.
 
 ---
 
@@ -500,4 +501,3 @@ To report a security vulnerability, see [docs/SECURITY.md](docs/SECURITY.md).
 
 MIT License. See [LICENSE](LICENSE) for details.
 
-

{voidaccess-1.7.1 → voidaccess-1.7.2}/api/routes/actors.py

@@ -22,7 +22,7 @@ from typing import Optional
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from pydantic import BaseModel, Field
-from sqlalchemy import or_
+from sqlalchemy import or_, select
 
 from api.auth import CurrentUser, get_current_user
 
@@ -88,26 +88,22 @@ def _db_required() -> None:
         raise HTTPException(status_code=503, detail="Database not configured")
 
 
-def _user_owned_investigation_ids_subq(session, user_id: int):
+def _user_owned_investigation_ids_select(user_id: int):
     from db.models import Investigation
 
-    return (
-        session.query(Investigation.id)
-        .filter(Investigation.user_id == user_id)
-        .subquery()
-    )
+    return select(Investigation.id).where(Investigation.user_id == user_id)
 
 
 def _profile_accessible(session, profile_id: str, user_id: int) -> bool:
     """True when at least one investigation linked to the profile is owned by the user."""
     from db.models import ActorAlias, ActorInfrastructure
 
-    inv_ids_subq = _user_owned_investigation_ids_subq(session, user_id)
+    inv_ids_select = _user_owned_investigation_ids_select(user_id)
     alias_match = (
         session.query(ActorAlias.id)
         .filter(
             ActorAlias.actor_id == profile_id,
-            ActorAlias.source_investigation_id.in_(inv_ids_subq),
+            ActorAlias.source_investigation_id.in_(inv_ids_select),
         )
         .first()
     )
@@ -117,7 +113,7 @@ def _profile_accessible(session, profile_id: str, user_id: int) -> bool:
         session.query(ActorInfrastructure.id)
         .filter(
             ActorInfrastructure.actor_id == profile_id,
-            ActorInfrastructure.source_investigation_id.in_(inv_ids_subq),
+            ActorInfrastructure.source_investigation_id.in_(inv_ids_select),
         )
         .first()
     )
@@ -146,8 +142,6 @@ async def list_actors(
     users since the seed is public data.
     """
     _db_required()
-    from sqlalchemy import select
-
     from db.models import (
         ActorProfile,
         ActorAlias,
@@ -156,9 +150,7 @@ async def list_actors(
 
     try:
         with _session() as session:
-            user_inv_ids = _user_owned_investigation_ids_subq(
-                session, current_user.user.id
-            )
+            user_inv_ids = _user_owned_investigation_ids_select(current_user.user.id)
 
             # Subquery of profile IDs visible to this user
             visible_alias = select(ActorAlias.actor_id).where(
@@ -176,13 +168,13 @@ async def list_actors(
             )
             if search:
                 pattern = f"%{search.lower()}%"
-                alias_subq = select(ActorAlias.actor_id).where(
+                alias_select = select(ActorAlias.actor_id).where(
                     ActorAlias.alias_value.ilike(pattern)
                 )
                 q = q.filter(
                     or_(
                         ActorProfile.canonical_handle.ilike(pattern),
-                        ActorProfile.id.in_(alias_subq),
+                        ActorProfile.id.in_(alias_select),
                     )
                 )
 
@@ -313,9 +305,7 @@ async def get_actor_investigations(
 
     try:
         with _session() as session:
-            user_inv_ids = _user_owned_investigation_ids_subq(
-                session, current_user.user.id
-            )
+            user_inv_ids = _user_owned_investigation_ids_select(current_user.user.id)
             all_ids = [
                 uuid_value(s)
                 for s in profile.get("investigation_ids", [])

{voidaccess-1.7.1 → voidaccess-1.7.2}/api/routes/entities.py

@@ -57,14 +57,12 @@ async def list_entities(
 
         with get_session() as session:
             user_inv_ids = (
-                session.query(Investigation.id)
-                .filter(Investigation.user_id == current_user.user.id)
-                .subquery()
+                sa.select(Investigation.id)
+                .where(Investigation.user_id == current_user.user.id)
             )
             linked_entity_ids = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
-                .subquery()
+                sa.select(InvestigationEntityLink.entity_id)
+                .where(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
             )
             q = session.query(Entity).filter(
                 sa.or_(
@@ -749,14 +747,12 @@ def _assert_entity_accessible(session, entity_id: uuid.UUID, user_id: int) -> No
     from db.models import Entity, Investigation, InvestigationEntityLink  # noqa: PLC0415
 
     user_inv_ids = (
-        session.query(Investigation.id)
-        .filter(Investigation.user_id == user_id)
-        .subquery()
+        sa.select(Investigation.id)
+        .where(Investigation.user_id == user_id)
     )
     linked_entity_ids = (
-        session.query(InvestigationEntityLink.entity_id)
-        .filter(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
-        .subquery()
+        sa.select(InvestigationEntityLink.entity_id)
+        .where(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
     )
     accessible = (
         session.query(Entity.id)

{voidaccess-1.7.1 → voidaccess-1.7.2}/api/routes/investigations.py

@@ -672,30 +672,30 @@ def _get_db_investigation(investigation_id: str) -> Any:
             )
 
             # Entity IDs for this investigation = own entities + junction-table links
-            linked_ids_subq = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id == inv.id)
-                .subquery()
-            )
-            entity_subq = (
-                session.query(Entity.id)
-                .filter(
-                    (Entity.investigation_id == inv.id)
-                    | Entity.id.in_(linked_ids_subq)
-                )
-                .subquery()
-            )
-            entity_count = int(
-                session.query(func.count()).select_from(entity_subq).scalar() or 0
-            )
-            relationship_count = int(
-                session.query(func.count(EntityRelationship.id))
-                .filter(
-                    (EntityRelationship.entity_a_id.in_(entity_subq))
-                    | (EntityRelationship.entity_b_id.in_(entity_subq))
-                )
-                .scalar()
-                or 0
+            linked_ids_select = (
+                sa_select(InvestigationEntityLink.entity_id)
+                .where(InvestigationEntityLink.investigation_id == inv.id)
+            )
+            entity_subq = (
+                session.query(Entity.id)
+                .filter(
+                    (Entity.investigation_id == inv.id)
+                    | Entity.id.in_(linked_ids_select)
+                )
+                .subquery()
+            )
+            entity_ids_select = sa_select(entity_subq.c.id)
+            entity_count = int(
+                session.query(func.count()).select_from(entity_subq).scalar() or 0
+            )
+            relationship_count = int(
+                session.query(func.count(EntityRelationship.id))
+                .filter(
+                    (EntityRelationship.entity_a_id.in_(entity_ids_select))
+                    | (EntityRelationship.entity_b_id.in_(entity_ids_select))
+                )
+                .scalar()
+                or 0
             )
 
             # Phase 6.1 — sources_used / infrastructure_clusters from DB metadata.
@@ -2673,14 +2673,13 @@ async def get_investigation_entities(
             if str(inv.user_id) != str(current_user.user.id):
                 raise HTTPException(status_code=403, detail="Forbidden")
 
-            linked_ids_subq = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id == inv.id)
-                .subquery()
-            )
+            linked_ids_select = (
+                sa_select(InvestigationEntityLink.entity_id)
+                .where(InvestigationEntityLink.investigation_id == inv.id)
+            )
             query = session.query(Entity).filter(
                 (Entity.investigation_id == inv.id)
-                | Entity.id.in_(linked_ids_subq)
+                | Entity.id.in_(linked_ids_select)
             )
             if entity_type:
                 query = query.filter(Entity.entity_type == entity_type)
@@ -2800,16 +2799,15 @@ async def export_investigation_entities_csv(
             if str(inv.user_id) != str(current_user.user.id):
                 raise HTTPException(status_code=403, detail="Forbidden")
 
-            linked_ids_subq = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id == inv.id)
-                .subquery()
-            )
+            linked_ids_select = (
+                sa_select(InvestigationEntityLink.entity_id)
+                .where(InvestigationEntityLink.investigation_id == inv.id)
+            )
             entities = (
                 session.query(Entity)
                 .filter(
                     (Entity.investigation_id == inv.id)
-                    | Entity.id.in_(linked_ids_subq)
+                    | Entity.id.in_(linked_ids_select)
                 )
                 .all()
             )

{voidaccess-1.7.1 → voidaccess-1.7.2}/api/routes/search.py

@@ -138,14 +138,12 @@ async def search_entities(
 
         with get_session() as session:
             user_inv_ids = (
-                session.query(Investigation.id)
-                .filter(Investigation.user_id == current_user.user.id)
-                .subquery()
+                sa.select(Investigation.id)
+                .where(Investigation.user_id == current_user.user.id)
             )
             linked_entity_ids = (
-                session.query(InvestigationEntityLink.entity_id)
-                .filter(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
-                .subquery()
+                sa.select(InvestigationEntityLink.entity_id)
+                .where(InvestigationEntityLink.investigation_id.in_(user_inv_ids))
             )
             q = session.query(Entity).filter(
                 sa.or_(

{voidaccess-1.7.1 → voidaccess-1.7.2}/config.py

@@ -6,6 +6,7 @@ from dotenv import load_dotenv
 load_dotenv()
 
 logger = logging.getLogger(__name__)
+_OPTIONAL_CONFIG_WARNING_SHOWN = False
 
 
 def _clean_env(name, default=None):
@@ -183,19 +184,24 @@ OPTIONAL_KEYS = [
 
 
 def validate_config():
+    global _OPTIONAL_CONFIG_WARNING_SHOWN
     missing_required = []
     for key in REQUIRED_KEYS:
         if _clean_env(key) is None:
             missing_required.append(key)
     if missing_required:
         raise RuntimeError(f"Missing required configuration keys: {', '.join(missing_required)}")
-    for key in OPTIONAL_KEYS:
-        val = _clean_env(key)
-        if val is None or val == "":
-            logger.warning(
-                "Optional configuration key '%s' is not set - related features will be disabled",
-                key,
-            )
+    missing_optional = [
+        key for key in OPTIONAL_KEYS
+        if (_clean_env(key) is None or _clean_env(key) == "")
+    ]
+    if missing_optional and not _OPTIONAL_CONFIG_WARNING_SHOWN:
+        logger.warning(
+            "Optional configuration keys not set (%d): %s - related features will be disabled",
+            len(missing_optional),
+            ", ".join(missing_optional),
+        )
+        _OPTIONAL_CONFIG_WARNING_SHOWN = True
 
 
 validate_config()

{voidaccess-1.7.1 → voidaccess-1.7.2}/db/queries.py

@@ -69,16 +69,15 @@ def count_distinct_pages_for_investigation(
     junction table (deduped entities from previous investigations re-linked here).
     """
     from db.models import InvestigationEntityLink  # noqa: PLC0415
-    linked_ids_subq = (
-        session.query(InvestigationEntityLink.entity_id)
-        .filter(InvestigationEntityLink.investigation_id == investigation_id)
-        .subquery()
+    linked_ids_select = (
+        sa.select(InvestigationEntityLink.entity_id)
+        .where(InvestigationEntityLink.investigation_id == investigation_id)
     )
     n = (
         session.query(sa.func.count(sa.distinct(Entity.page_id)))
         .filter(
             (Entity.investigation_id == investigation_id)
-            | Entity.id.in_(linked_ids_subq)
+            | Entity.id.in_(linked_ids_select)
         )
         .scalar()
     )

{voidaccess-1.7.1 → voidaccess-1.7.2}/export/ioc_package.py

@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.7.1"
+PACKAGE_VERSION = "1.7.2"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.7.1 → voidaccess-1.7.2}/export/misp.py

@@ -192,16 +192,15 @@ def _load_investigation_and_entities(
             if investigation is None:
                 return None, []
 
-            linked_ids_subq = (
+            linked_ids_select = (
                 session.query(InvestigationEntityLink.entity_id)
                 .filter(InvestigationEntityLink.investigation_id == investigation.id)
-                .subquery()
             )
             db_entities = (
                 session.query(Entity)
                 .filter(
                     (Entity.investigation_id == investigation.id)
-                    | Entity.id.in_(linked_ids_subq)
+                    | Entity.id.in_(linked_ids_select)
                 )
                 .all()
             )

{voidaccess-1.7.1 → voidaccess-1.7.2}/export/sigma.py

@@ -294,16 +294,15 @@ def _load_entities_for_investigation(investigation_id: Any) -> list[Any]:
             if inv is None:
                 return []
 
-            linked_ids_subq = (
+            linked_ids_select = (
                 session.query(InvestigationEntityLink.entity_id)
                 .filter(InvestigationEntityLink.investigation_id == inv.id)
-                .subquery()
             )
             db_entities = (
                 session.query(Entity)
                 .filter(
                     (Entity.investigation_id == inv.id)
-                    | Entity.id.in_(linked_ids_subq)
+                    | Entity.id.in_(linked_ids_select)
                 )
                 .all()
             )

{voidaccess-1.7.1 → voidaccess-1.7.2}/export/stix.py

@@ -63,6 +63,9 @@ _STIX_PATTERNS: dict[str, str] = {
     "ETHEREUM_ADDRESS":  "[cryptocurrency-wallet:address = '{value}']",
     "MONERO_ADDRESS":    "[cryptocurrency-wallet:address = '{value}']",
     "EMAIL_ADDRESS":     "[email-message:from_ref.value = '{value}']",
+    "FILE_HASH_MD5":     "[file:hashes.MD5 = '{value}']",
+    "FILE_HASH_SHA1":    "[file:hashes.'SHA-1' = '{value}']",
+    "FILE_HASH_SHA256":  "[file:hashes.'SHA-256' = '{value}']",
     "ONION_URL":         "[url:value = '{value}']",
     "IP_ADDRESS":        "[ipv4-addr:value = '{value}']",
     "CVE_NUMBER":        "[vulnerability:name = '{value}']",
@@ -111,6 +114,8 @@ def entity_to_stix_indicator(entity: Any) -> Optional[Any]:
         indicator_types = ["malicious-activity"]
     elif etype in ("BITCOIN_ADDRESS", "ETHEREUM_ADDRESS", "MONERO_ADDRESS"):
         indicator_types = ["malicious-activity"]
+    elif etype in ("FILE_HASH_MD5", "FILE_HASH_SHA1", "FILE_HASH_SHA256"):
+        indicator_types = ["malicious-activity"]
     elif etype in ("IP_ADDRESS", "ONION_URL"):
         indicator_types = ["malicious-activity"]
     elif etype == "CVE_NUMBER":
@@ -424,6 +429,7 @@ def _build_stix_relationships(
         graph = build_graph_from_db(investigation_id=inv_uuid)
 
         relationships: list[Any] = []
+        seen_relationships: set[tuple[str, str, str]] = set()
         for source_node, target_node, data in graph.edges(data=True):
             src_stix_id = stix_id_map.get(source_node)
             tgt_stix_id = stix_id_map.get(target_node)
@@ -432,6 +438,10 @@ def _build_stix_relationships(
             edge_type = data.get("edge_type", "related-to")
             # Map VoidAccess edge types to STIX relationship types
             rel_type = _edge_type_to_stix(edge_type)
+            relationship_key = (rel_type, src_stix_id, tgt_stix_id)
+            if relationship_key in seen_relationships:
+                continue
+            seen_relationships.add(relationship_key)
             try:
                 rel = stix2.Relationship(
                     relationship_type=rel_type,

{voidaccess-1.7.1 → voidaccess-1.7.2}/extractor/normalizer.py

@@ -623,6 +623,11 @@ ENTITY_BLOCKLIST: frozenset[str] = frozenset({
     "moderator", "administrator", "operator", "staff", "support",
     "customer", "vendor", "buyer", "seller", "trader",
     "dropper", "loader", "stager", "payload", "beacon",
+    # Audit-confirmed NER false positives (1.6.2, 1.7.0, 1.7.1).
+    "asks", "keyword", "sends", "input", "content", "delivery", "into",
+    "submits", "advertising", "attempts", "pass", "consent", "devices",
+    "stdin", "stdout", "detect", "uri_regex:1", "intel", "passive",
+    "commands",
 })
 
 KNOWN_TOOLS: frozenset[str] = frozenset({
@@ -633,7 +638,15 @@ KNOWN_TOOLS: frozenset[str] = frozenset({
     "kali", "parrot", "blackarch", "backtrack",
 })
 
+COMMON_THREAT_TERMS: frozenset[str] = frozenset({
+    "apt41", "apt28", "apt29", "fin7", "fin8", "lockbitsupp",
+    "scatteredspider", "lazarus", "sandworm", "blackcat", "alphv",
+})
+
 LEET_GENERIC = re.compile(r"^h[4a][ck]+[3e]?r?$")
+HANDLE_SHAPE_RE = re.compile(
+    r"^(?:[A-Z]{2,}\d{1,3}|[A-Za-z]+[A-Z][A-Za-z0-9]*|[A-Za-z0-9]+[_\-.][A-Za-z0-9_.-]+|[A-Za-z]+\d+[A-Za-z0-9]*)$"
+)
 
 
 ENTITY_MIN_LENGTH: dict[str, int] = {
@@ -723,6 +736,13 @@ def is_blocked_entity(entity_type: str, entity_value: str) -> bool:
             return True
         if LEET_GENERIC.match(value_lower):
             return True
+        if value_lower in COMMON_THREAT_TERMS:
+            return False
+        if re.fullmatch(r"[a-z]+", value_lower) and not HANDLE_SHAPE_RE.match(entity_value.strip()):
+            return True
+
+    if entity_type == "ORGANIZATION_NAME" and value_lower in ENTITY_BLOCKLIST:
+        return True
 
     min_len = ENTITY_MIN_LENGTH.get(entity_type, 3)
     if len(value_lower) < min_len:
@@ -868,13 +888,87 @@ _ORG_NOISE_RE = re.compile(
     r'|[<>{}\[\]|\\]',
     re.IGNORECASE,
 )
+_HTML_ENTITY_RE = re.compile(r"&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);?", re.IGNORECASE)
+_DECODED_HTML_UI_RE = re.compile(r"[\u2190-\u21ff\u00ae\u00a9\u2122]")
+_ORG_SENTENCE_WORDS: frozenset[str] = frozenset({
+    "the", "a", "an", "how", "cancel", "reply", "post", "posts", "latest",
+    "popular", "tutorial", "tutorials", "guide", "home", "news", "security",
+    "advertisement", "advertising", "speaking", "subscribe", "mailing",
+    "list", "leave", "recent", "author", "actively", "exploited", "analyzing",
+    "targeted", "intrusions", "through", "stay", "upcoming", "seizes",
+    "related", "articles", "dummies", "vulnerability", "date", "added",
+    "expands", "leadership", "approach", "radar", "supply", "chain",
+    "compromise", "commercial", "live", "proxies", "software", "office",
+    "oauth", "os", "application", "server", "protect", "sharepoint",
+    "tv", "stealer", "stack", "protection", "organization", "redacted",
+    "organizations", "are", "building", "achieves", "patches", "moves",
+    "contact", "rule", "addition", "activated", "scale", "automation",
+    "algorithmic", "payload", "decomposition", "third-party", "third",
+    "party", "additional", "resources", "learn", "join", "dll",
+    "sideloading", "factory", "malware", "threat", "research", "tags",
+    "copilot", "next-gen", "xmrig", "authenticode", "updatefactor",
+    "coldfusion", "falcon", "openid", "overwatch", "analysis", "table",
+    "antimalware", "scan", "interface", "bypass", "static", "file-size",
+    "inflation", "loaders", "data", "exe", "getassertion",
+    "getcredentials", "imphash", "cluster", "role", "remote", "desktop",
+    "protocol", "sha-256", "webauthn", "yubikey", "systemagentservice",
+    "chromium", "clientdatahash", "gift", "card", "fraud", "macos",
+    "linux", "tools", "typescript", "webassembly", "webhid",
+    "ai-hallucinated", "hallucinated", "domains", "comparison", "llm",
+    "dataset", "jaccard", "multi-signal", "risk", "classification",
+    "share", "executive", "summary", "temperature", "configuration",
+    "account", "numbers", "balanced", "llm2", "briefs",
+})
+_ORG_ALLOWED_CONNECTORS: frozenset[str] = frozenset({
+    "and", "of", "for", "to", "in", "on", "with",
+})
+_ORG_SUFFIX_HINTS: frozenset[str] = frozenset({
+    "agency", "bank", "bureau", "center", "centre", "committee", "company",
+    "corp", "corporation", "council", "department", "directorate", "division",
+    "foundation", "gmbh", "group", "inc", "institute", "labs", "limited",
+    "llc", "ltd", "networks", "team", "technologies", "university", "union",
+})
 
 
 def _is_valid_org_name(value: str) -> bool:
     v = value.strip()
+    v_lower = v.lower()
+    if v_lower in ENTITY_BLOCKLIST:
+        return False
     if len(v) < 3 or len(v) > 60:
         return False
-    if len(v.split()) > 5:
+    if re.search(r"[:,/()]", v):
+        return False
+    if re.search(r"\.[a-z0-9]{2,5}\b", v_lower):
+        return False
+    if re.search(r"\d{1,2}/\d{1,2}/\d{2,4}", v):
+        return False
+    if _HTML_ENTITY_RE.search(v) or _DECODED_HTML_UI_RE.search(v):
+        return False
+    if re.search(r"\b[a-z]+['\u2019]\w*\b", v_lower):
+        return False
+    tokens = re.findall(r"[a-z0-9]+(?:[-.][a-z0-9]+)*", v_lower)
+    if not tokens:
+        return False
+    if any(token.isdigit() for token in tokens):
+        return False
+    if any(token == "cve" or token.startswith("cve-") for token in tokens):
+        return False
+    if len(tokens) > 4:
+        return False
+    if "-" in v and not any(token in _ORG_SUFFIX_HINTS for token in tokens):
+        return False
+    if any(token in _ORG_SENTENCE_WORDS for token in tokens):
+        return False
+    word_tokens = [token for token in tokens if re.search(r"[a-z]", token)]
+    if len(word_tokens) > 1:
+        generic_words = [
+            token for token in word_tokens
+            if token in ENTITY_BLOCKLIST or token in _ORG_ALLOWED_CONNECTORS
+        ]
+        if len(generic_words) >= len(word_tokens):
+            return False
+    if re.fullmatch(r"[A-Z_]+:?\d*", v) and v_lower not in COMMON_THREAT_TERMS:
         return False
     if _ORG_NOISE_RE.search(v):
         return False

{voidaccess-1.7.1 → voidaccess-1.7.2}/extractor/pipeline.py

@@ -19,6 +19,7 @@ ExtractionResult is a dataclass exported through extractor/__init__.py.
 from __future__ import annotations
 
 import asyncio
+import html
 import logging
 from dataclasses import dataclass, field
 from typing import Any, Optional
@@ -175,6 +176,13 @@ def _type_priority(entity_type: str) -> int:
             return priority
     return 99
 
+
+def _clean_text_for_extraction(text: str) -> str:
+    """Decode HTML entities before regex/NER/LLM extraction sees page text."""
+    if not text:
+        return ""
+    return html.unescape(text)
+
 # ---------------------------------------------------------------------------
 # Result dataclass
 # ---------------------------------------------------------------------------
@@ -231,6 +239,7 @@ async def extract_entities_from_page(
     by regex, or below the top-N priority cut).
     """
     errors: list[str] = []
+    page_text = _clean_text_for_extraction(page_text or "")
 
     # -----------------------------------------------------------------------
     # Stage 1 — Regex