voidaccess 2.0.1__tar.gz → 2.0.3__tar.gz

{voidaccess-2.0.1 → voidaccess-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 2.0.1
+Version: 2.0.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-2.0.1 → voidaccess-2.0.3}/api/main.py

@@ -332,7 +332,7 @@ async def _periodic_stuck_sweep() -> None:
 app = FastAPI(
     title="VoidAccess Intelligence API",
     description="VoidAccess: Dark Web Intelligence Platform",
-    version="2.0.1",
+    version="2.0.3",
     docs_url="/docs",
     redoc_url="/redoc",
     lifespan=lifespan,

{voidaccess-2.0.1 → voidaccess-2.0.3}/export/__init__.py

@@ -17,6 +17,7 @@ from export.misp import (
 from export.sigma import (
     entities_to_sigma_rules,
     export_sigma_rules,
+    render_sigma_rules,
     sigma_rule_to_yaml,
 )
 from export.yara_export import (
@@ -44,6 +45,7 @@ __all__ = [
     # sigma
     "entities_to_sigma_rules",
     "sigma_rule_to_yaml",
+    "render_sigma_rules",
     "export_sigma_rules",
     # yara
     "generate_yara_rules",

{voidaccess-2.0.1 → voidaccess-2.0.3}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "2.0.1"
+PACKAGE_VERSION = "2.0.3"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.
@@ -702,57 +702,13 @@ def _safe_misp_json(entities: list[Any], investigation_id: Any) -> str:
 
 def _safe_sigma_yml(entities: list[Any]) -> str:
     try:
-        from export.sigma import entities_to_sigma_rules, sigma_rule_to_yaml
-        # Sigma rules expect object-style entities (NormalizedEntity).  When the
-        # caller passes plain dicts, wrap them in a tiny adapter so Sigma can
-        # read .entity_type, .value, .confidence, and .source_url.
-        adapted: list[Any] = []
-        for e in entities or []:
-            if isinstance(e, dict):
-                adapted.append(_DictEntityAdapter(e))
-            else:
-                adapted.append(e)
-        rules = entities_to_sigma_rules(adapted)
-        return "\n---\n".join(sigma_rule_to_yaml(r) for r in rules if r)
+        from export.sigma import render_sigma_rules
+        return render_sigma_rules(entities or [])
     except Exception as exc:
         logger.warning("Sigma generation for IOC package failed: %s", exc)
         return ""
 
 
-class _DictEntityAdapter:
-    """Adapt a plain dict into an object with .entity_type / .value / etc.
-
-    Used to feed dict-shaped entities into Sigma / STIX / MISP generators
-    that expect object access.  Keeps the IOC package module standalone
-    without changing the existing Sigma / STIX / MISP module signatures.
-    """
-
-    __slots__ = ("_d",)
-
-    def __init__(self, d: dict[str, Any]) -> None:
-        self._d = d
-
-    @property
-    def entity_type(self) -> str:
-        return _entity_type(self._d)
-
-    @property
-    def value(self) -> str:
-        return _entity_value(self._d)
-
-    @property
-    def canonical_value(self) -> str:
-        return entity_canonical_id(self._d)
-
-    @property
-    def confidence(self) -> float:
-        return _entity_confidence(self._d)
-
-    @property
-    def source_url(self) -> str:
-        return _entity_source_url(self._d)
-
-
 # ---------------------------------------------------------------------------
 # README + metadata builders
 # ---------------------------------------------------------------------------

{voidaccess-2.0.1 → voidaccess-2.0.3}/export/sigma.py

@@ -207,7 +207,8 @@ def entities_to_sigma_rules(
     Falls back to base rule if LLM fails.
     """
     rules: list[dict] = []
-    for entity in entities:
+    for raw_entity in entities:
+        entity = _adapt_entity(raw_entity)
         if entity.entity_type not in _SIGMA_ENTITY_TYPES:
             continue
         base = _build_base_rule(entity)
@@ -228,6 +229,23 @@ def sigma_rule_to_yaml(rule: dict) -> str:
         return ""
 
 
+def render_sigma_rules(
+    entities: list[Any],
+    llm: Optional[Any] = None,
+) -> str:
+    """Render the shared Sigma artifact used by every export surface.
+
+    The CLI export and the IOC package both receive the same export payload,
+    whose entities are normally dictionaries.  Keeping adaptation, rule
+    generation, and YAML assembly here prevents one surface from growing a
+    second DB-loading implementation with different failure semantics.
+    """
+    rules = entities_to_sigma_rules(list(entities or []), llm=llm)
+    return "\n---\n".join(
+        sigma_rule_to_yaml(rule) for rule in rules if rule
+    )
+
+
 def export_sigma_rules(
     investigation_id: Any,
     output_dir: str,
@@ -320,6 +338,45 @@ def _load_entities_for_investigation(investigation_id: Any) -> list[Any]:
         return []
 
 
+class _DictEntityAdapter:
+    """Expose dictionary-shaped export entities through Sigma's object API."""
+
+    __slots__ = ("_data",)
+
+    def __init__(self, data: dict[str, Any]) -> None:
+        self._data = data
+
+    @property
+    def entity_type(self) -> str:
+        return str(self._data.get("entity_type") or "")
+
+    @property
+    def value(self) -> str:
+        return str(
+            self._data.get("canonical_value")
+            or self._data.get("value")
+            or ""
+        )
+
+    @property
+    def confidence(self) -> float:
+        try:
+            return float(self._data.get("confidence", 0.0) or 0.0)
+        except (TypeError, ValueError):
+            return 0.0
+
+    @property
+    def source_url(self) -> str:
+        return str(self._data.get("source_url") or "")
+
+
+def _adapt_entity(entity: Any) -> Any:
+    """Accept both ORM/normalized entities and export dictionaries."""
+    if isinstance(entity, dict):
+        return _DictEntityAdapter(entity)
+    return entity
+
+
 def _coerce_uuid(value: Any):
     """Coerce value to uuid.UUID. Returns None on failure."""
     import uuid as _uuid

{voidaccess-2.0.1 → voidaccess-2.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "2.0.1"
+version = "2.0.3"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"
@@ -154,7 +154,7 @@ exclude = [
 "sources" = ["data/onion_seeds.json"]
 "sources.data" = ["onion_seeds.json"]
 
-# v2.0.1 — ship the top-level reference data bundle inside the installed
+# v2.0.2 — ship the top-level reference data bundle inside the installed
 # package.  The runtime resolves each of these via
 # ``os.path.dirname(os.path.dirname(__file__))/data`` (from extractor/ and
 # monitor/), i.e. ``site-packages/data/<file>`` after a wheel install.  The

{voidaccess-2.0.1 → voidaccess-2.0.3}/tests/test_cli_proxy_config.py

@@ -46,6 +46,8 @@ No real network.  All file I/O uses tmp_path.
 from __future__ import annotations
 
 import os
+import stat
+import subprocess
 import sys
 import types
 
@@ -64,11 +66,42 @@ def isolated_config_home(tmp_path, monkeypatch):
 
     monkeypatch.setattr(cli_config, "CLI_HOME", tmp_path)
     monkeypatch.setattr(cli_config, "CONFIG_PATH", tmp_path / "config.json")
+    monkeypatch.setattr(cli_config, "ENV_PATH", tmp_path / ".env")
     monkeypatch.setattr(cli_config, "DB_PATH", tmp_path / "investigations.db")
     monkeypatch.setattr(cli_config, "DEFAULT_OUTPUT_DIR", tmp_path / "results")
     return cli_config
 
 
+def test_fresh_configure_run_locks_env_file(isolated_config_home):
+    """A configure write applies the same owner-only protection to .env."""
+    isolated_config_home.ENV_PATH.write_text(
+        "JWT_SECRET=test-only\n",
+        encoding="utf-8",
+    )
+
+    from typer.testing import CliRunner
+    from voidaccess_cli.commands.configure import app as configure_app
+
+    result = CliRunner().invoke(
+        configure_app,
+        ["tor", "--host", "127.0.0.1", "--port", "9050"],
+    )
+    assert result.exit_code == 0, result.output
+
+    if os.name != "nt":
+        assert stat.S_IMODE(isolated_config_home.ENV_PATH.stat().st_mode) == 0o600
+    else:
+        acl = subprocess.run(
+            ["icacls", str(isolated_config_home.ENV_PATH)],
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout
+        assert "(I)" not in acl
+        assert "BUILTIN\\Users" not in acl
+        assert "Authenticated Users" not in acl
+
+
 def _scrub_env(monkeypatch):
     """Remove every proxy-related env var so each test starts from a known state."""
     for k in (

voidaccess-2.0.3/tests/test_legacy_backfill.py

@@ -0,0 +1,124 @@
+"""Tests for the one-time v2.0.3 legacy-data repair."""
+
+from __future__ import annotations
+
+import datetime
+import uuid
+
+
+def test_backfill_resolves_typed_page_and_persists_communities(db_engine):
+    from db.models import Entity, EntityRelationship, Investigation, Page
+    from db.session import get_session
+    from voidaccess.backfill import run_backfill
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    investigation_id = uuid.uuid4()
+    page_id = uuid.uuid4()
+    source_id, target_id = uuid.uuid4(), uuid.uuid4()
+
+    with get_session() as session:
+        session.add(
+            Investigation(
+                id=investigation_id,
+                query="legacy backfill test",
+                status="completed",
+                graph_status="built",
+                created_at=now,
+            )
+        )
+        session.add(
+            Page(
+                id=page_id,
+                url="https://example.test/legacy",
+                cleaned_text="Actor uses malware",
+                scrape_timestamp=now,
+                created_at=now,
+            )
+        )
+        session.add_all(
+            [
+                Entity(
+                    id=source_id,
+                    page_id=page_id,
+                    investigation_id=investigation_id,
+                    entity_type="THREAT_ACTOR_HANDLE",
+                    value="actor-one",
+                    canonical_value="actor-one",
+                    confidence=0.95,
+                    first_seen=now,
+                    last_seen=now,
+                    created_at=now,
+                ),
+                Entity(
+                    id=target_id,
+                    page_id=page_id,
+                    investigation_id=investigation_id,
+                    entity_type="MALWARE_FAMILY",
+                    value="malware-one",
+                    canonical_value="malware-one",
+                    confidence=0.95,
+                    first_seen=now,
+                    last_seen=now,
+                    created_at=now,
+                ),
+            ]
+        )
+        session.add(
+            EntityRelationship(
+                entity_a_id=source_id,
+                entity_b_id=target_id,
+                relationship_type="USES",
+                confidence=0.9,
+                investigation_id=investigation_id,
+                source_page_id=None,
+                first_seen=now,
+            )
+        )
+        session.commit()
+
+        result = run_backfill(session, investigation_id)
+        session.commit()
+
+        relationship = session.query(EntityRelationship).one()
+        investigation = session.get(Investigation, investigation_id)
+
+        assert result["typed_relationship_pages"]["resolved"] == 1
+        assert relationship.source_page_id == page_id
+        assert investigation.metadata_json["communities"]
+        assert investigation.metadata_json["community_count"] == 1
+
+
+def test_backfill_does_not_assign_invalid_typed_relationship_page(db_engine):
+    from db.models import Entity, EntityRelationship, Investigation, Page
+    from db.session import get_session
+    from voidaccess.backfill import backfill_typed_relationship_page_ids
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    investigation_id = uuid.uuid4()
+    page_id = uuid.uuid4()
+    source_id, target_id = uuid.uuid4(), uuid.uuid4()
+
+    with get_session() as session:
+        session.add(
+            Investigation(
+                id=investigation_id,
+                query="invalid legacy relationship",
+                status="completed",
+                created_at=now,
+            )
+        )
+        session.add(Page(id=page_id, url="https://example.test/invalid", scrape_timestamp=now, created_at=now))
+        session.add_all(
+            [
+                Entity(id=source_id, page_id=page_id, investigation_id=investigation_id,
+                       entity_type="ORGANIZATION_NAME", value="org", canonical_value="org"),
+                Entity(id=target_id, page_id=page_id, investigation_id=investigation_id,
+                       entity_type="IP_ADDRESS", value="192.0.2.1", canonical_value="192.0.2.1"),
+                EntityRelationship(entity_a_id=source_id, entity_b_id=target_id,
+                                   relationship_type="TARGETS", investigation_id=investigation_id,
+                                   first_seen=now),
+            ]
+        )
+        session.commit()
+        result = backfill_typed_relationship_page_ids(session, investigation_id)
+        assert result == {"scanned": 1, "resolved": 0, "unresolved": 1}

voidaccess-2.0.3/voidaccess/backfill.py

@@ -0,0 +1,251 @@
+"""One-time repairs for legacy investigation data.
+
+This module deliberately does not change the schema.  It repairs two pieces of
+derived data that were introduced after some investigations had already been
+stored: the page provenance on typed relationship edges and the persisted
+community partition used by the HTTP graph endpoint.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import uuid
+from collections import defaultdict
+from typing import Any, Optional
+
+from sqlalchemy.orm import joinedload
+
+from extractor.relationship_extract import _LLM_REL_VOCAB, _is_compatible_relationship
+from graph.builder import build_graph_from_db, detect_communities
+
+logger = logging.getLogger(__name__)
+
+
+def _as_uuid(value: Any) -> Optional[uuid.UUID]:
+    if isinstance(value, uuid.UUID):
+        return value
+    try:
+        return uuid.UUID(str(value)) if value else None
+    except (TypeError, ValueError, AttributeError):
+        return None
+
+
+def _metadata_dict(value: Any) -> dict[str, Any]:
+    if isinstance(value, dict):
+        return dict(value)
+    if isinstance(value, str) and value:
+        try:
+            decoded = json.loads(value)
+        except (TypeError, ValueError):
+            return {}
+        return dict(decoded) if isinstance(decoded, dict) else {}
+    return {}
+
+
+def _validated_entities(session, investigation_id: uuid.UUID):
+    """Return the current entity set, including cross-investigation links."""
+    from db.models import Entity, InvestigationEntityLink
+
+    return (
+        session.query(Entity)
+        .outerjoin(
+            InvestigationEntityLink,
+            InvestigationEntityLink.entity_id == Entity.id,
+        )
+        .filter(
+            (Entity.investigation_id == investigation_id)
+            | (InvestigationEntityLink.investigation_id == investigation_id)
+        )
+        .options(joinedload(Entity.page))
+        .distinct()
+        .all()
+    )
+
+
+def _resolve_source_page_id(
+    relationship,
+    entity_a,
+    entity_b,
+    validated_entities: list,
+) -> Optional[uuid.UUID]:
+    """Resolve provenance using the live pipeline's validated entity set.
+
+    Typed relationship direction is meaningful: the live pipeline persists the
+    page on which the claim was extracted.  For a legacy row without that
+    field, the source endpoint's page is the strongest deterministic evidence;
+    the target page and a page containing both endpoints are conservative
+    fallbacks.  No page outside the validated investigation set is accepted.
+    """
+    if not _is_compatible_relationship(
+        str(relationship.relationship_type or ""),
+        {"type": entity_a.entity_type},
+        {"type": entity_b.entity_type},
+    ):
+        return None
+
+    valid_page_ids = {
+        ent.page_id for ent in validated_entities if ent.page_id is not None
+    }
+    if not valid_page_ids:
+        return None
+
+    page_entity_ids: dict[uuid.UUID, set[uuid.UUID]] = defaultdict(set)
+    for ent in validated_entities:
+        if ent.page_id is not None:
+            page_entity_ids[ent.page_id].add(ent.id)
+
+    # The source endpoint is the direct equivalent of the page_id attached by
+    # relationship_extract.extract_relationships_from_results().
+    candidates = [
+        entity_a.page_id,
+        entity_b.page_id,
+    ]
+    for page_id, entity_ids in page_entity_ids.items():
+        if entity_a.id in entity_ids and entity_b.id in entity_ids:
+            candidates.append(page_id)
+    for page_id, entity_ids in page_entity_ids.items():
+        if entity_a.id in entity_ids:
+            candidates.append(page_id)
+
+    seen: set[uuid.UUID] = set()
+    for page_id in candidates:
+        if page_id is None or page_id in seen:
+            continue
+        seen.add(page_id)
+        if page_id in valid_page_ids:
+            return page_id
+    return None
+
+
+def backfill_typed_relationship_page_ids(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, int]:
+    """Populate missing ``source_page_id`` values on validated typed edges."""
+    from db.models import Entity, EntityRelationship
+
+    typed_types = set(_LLM_REL_VOCAB.values())
+    query = session.query(EntityRelationship).filter(
+        EntityRelationship.source_page_id.is_(None),
+        EntityRelationship.relationship_type.in_(typed_types),
+    )
+    if investigation_id is not None:
+        query = query.filter(EntityRelationship.investigation_id == investigation_id)
+
+    rows = query.options(
+        joinedload(EntityRelationship.entity_a).joinedload(Entity.page),
+        joinedload(EntityRelationship.entity_b).joinedload(Entity.page),
+    ).all()
+    entities_by_investigation: dict[uuid.UUID, list] = {}
+    scanned = resolved = unresolved = 0
+
+    for relationship in rows:
+        scanned += 1
+        entity_a = relationship.entity_a
+        entity_b = relationship.entity_b
+        target_investigation = (
+            _as_uuid(relationship.investigation_id)
+            or _as_uuid(getattr(entity_a, "investigation_id", None))
+            or _as_uuid(getattr(entity_b, "investigation_id", None))
+        )
+        if target_investigation is None:
+            unresolved += 1
+            continue
+        if target_investigation not in entities_by_investigation:
+            entities_by_investigation[target_investigation] = _validated_entities(
+                session, target_investigation
+            )
+        page_id = _resolve_source_page_id(
+            relationship,
+            entity_a,
+            entity_b,
+            entities_by_investigation[target_investigation],
+        )
+        if page_id is None:
+            unresolved += 1
+            continue
+        relationship.source_page_id = page_id
+        resolved += 1
+
+    if resolved:
+        session.flush()
+    return {"scanned": scanned, "resolved": resolved, "unresolved": unresolved}
+
+
+def backfill_community_partitions(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, int]:
+    """Persist detector output for completed investigations lacking a partition."""
+    from db.models import Investigation
+
+    query = session.query(Investigation).filter(Investigation.status == "completed")
+    if investigation_id is not None:
+        query = query.filter(Investigation.id == investigation_id)
+
+    scanned = persisted = skipped = 0
+    for investigation in query.order_by(Investigation.created_at).all():
+        scanned += 1
+        metadata = _metadata_dict(investigation.metadata_json)
+        existing = metadata.get("communities")
+        if isinstance(existing, dict) and existing:
+            skipped += 1
+            continue
+
+        graph = build_graph_from_db(investigation_id=investigation.id)
+        partition = {
+            str(node_id): int(community_id)
+            for node_id, community_id in detect_communities(graph).items()
+        }
+        metadata["communities"] = partition
+        metadata["community_count"] = len(set(partition.values())) if partition else 0
+        investigation.metadata_json = metadata
+        session.flush()
+        persisted += 1
+
+    return {"scanned": scanned, "persisted": persisted, "skipped": skipped}
+
+
+def run_backfill(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, dict[str, int]]:
+    """Run both repairs in one transaction."""
+    typed = backfill_typed_relationship_page_ids(session, investigation_id)
+    communities = backfill_community_partitions(session, investigation_id)
+    return {"typed_relationship_pages": typed, "community_partitions": communities}
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--db-path", help="SQLite database path (defaults to VOIDACCESS_DB_PATH)")
+    parser.add_argument("--investigation-id", help="Limit the repair to one investigation")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args(argv)
+
+    if args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    db_path = args.db_path or os.getenv("VOIDACCESS_DB_PATH")
+    if db_path:
+        db_path = os.path.abspath(os.path.expanduser(db_path))
+        os.environ["VOIDACCESS_DB_PATH"] = db_path
+        os.environ["DATABASE_URL"] = f"sqlite:///{db_path.replace(os.sep, '/') }"
+
+    target_id = _as_uuid(args.investigation_id)
+    if args.investigation_id and target_id is None:
+        parser.error("--investigation-id must be a UUID")
+
+    from db.session import get_session
+
+    with get_session() as session:
+        result = run_backfill(session, target_id)
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{voidaccess-2.0.1 → voidaccess-2.0.3}/voidaccess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 2.0.1
+Version: 2.0.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-2.0.1 → voidaccess-2.0.3}/voidaccess.egg-info/SOURCES.txt

@@ -186,6 +186,7 @@ tests/test_i18n.py
 tests/test_infostealer.py
 tests/test_ioc_package_export.py
 tests/test_ip_reputation.py
+tests/test_legacy_backfill.py
 tests/test_llm.py
 tests/test_llm_utils.py
 tests/test_model_singleton.py
@@ -232,6 +233,7 @@ vector/model_singleton.py
 vector/search.py
 vector/store.py
 voidaccess/__init__.py
+voidaccess/backfill.py
 voidaccess/config.py
 voidaccess/llm.py
 voidaccess/llm_utils.py

{voidaccess-2.0.1 → voidaccess-2.0.3}/voidaccess_cli/__init__.py

@@ -1,3 +1,3 @@
 """voidaccess CLI — dark-web OSINT command-line interface."""
 
-__version__ = "2.0.1"
+__version__ = "2.0.3"