voidaccess 2.0.2__tar.gz → 2.0.3__tar.gz

{voidaccess-2.0.2 → voidaccess-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 2.0.2
+Version: 2.0.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-2.0.2 → voidaccess-2.0.3}/api/main.py

@@ -332,7 +332,7 @@ async def _periodic_stuck_sweep() -> None:
 app = FastAPI(
     title="VoidAccess Intelligence API",
     description="VoidAccess: Dark Web Intelligence Platform",
-    version="2.0.2",
+    version="2.0.3",
     docs_url="/docs",
     redoc_url="/redoc",
     lifespan=lifespan,

{voidaccess-2.0.2 → voidaccess-2.0.3}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "2.0.2"
+PACKAGE_VERSION = "2.0.3"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-2.0.2 → voidaccess-2.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "2.0.2"
+version = "2.0.3"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"

{voidaccess-2.0.2 → voidaccess-2.0.3}/tests/test_cli_proxy_config.py

@@ -46,6 +46,8 @@ No real network.  All file I/O uses tmp_path.
 from __future__ import annotations
 
 import os
+import stat
+import subprocess
 import sys
 import types
 
@@ -64,11 +66,42 @@ def isolated_config_home(tmp_path, monkeypatch):
 
     monkeypatch.setattr(cli_config, "CLI_HOME", tmp_path)
     monkeypatch.setattr(cli_config, "CONFIG_PATH", tmp_path / "config.json")
+    monkeypatch.setattr(cli_config, "ENV_PATH", tmp_path / ".env")
     monkeypatch.setattr(cli_config, "DB_PATH", tmp_path / "investigations.db")
     monkeypatch.setattr(cli_config, "DEFAULT_OUTPUT_DIR", tmp_path / "results")
     return cli_config
 
 
+def test_fresh_configure_run_locks_env_file(isolated_config_home):
+    """A configure write applies the same owner-only protection to .env."""
+    isolated_config_home.ENV_PATH.write_text(
+        "JWT_SECRET=test-only\n",
+        encoding="utf-8",
+    )
+
+    from typer.testing import CliRunner
+    from voidaccess_cli.commands.configure import app as configure_app
+
+    result = CliRunner().invoke(
+        configure_app,
+        ["tor", "--host", "127.0.0.1", "--port", "9050"],
+    )
+    assert result.exit_code == 0, result.output
+
+    if os.name != "nt":
+        assert stat.S_IMODE(isolated_config_home.ENV_PATH.stat().st_mode) == 0o600
+    else:
+        acl = subprocess.run(
+            ["icacls", str(isolated_config_home.ENV_PATH)],
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout
+        assert "(I)" not in acl
+        assert "BUILTIN\\Users" not in acl
+        assert "Authenticated Users" not in acl
+
+
 def _scrub_env(monkeypatch):
     """Remove every proxy-related env var so each test starts from a known state."""
     for k in (

voidaccess-2.0.3/tests/test_legacy_backfill.py

@@ -0,0 +1,124 @@
+"""Tests for the one-time v2.0.3 legacy-data repair."""
+
+from __future__ import annotations
+
+import datetime
+import uuid
+
+
+def test_backfill_resolves_typed_page_and_persists_communities(db_engine):
+    from db.models import Entity, EntityRelationship, Investigation, Page
+    from db.session import get_session
+    from voidaccess.backfill import run_backfill
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    investigation_id = uuid.uuid4()
+    page_id = uuid.uuid4()
+    source_id, target_id = uuid.uuid4(), uuid.uuid4()
+
+    with get_session() as session:
+        session.add(
+            Investigation(
+                id=investigation_id,
+                query="legacy backfill test",
+                status="completed",
+                graph_status="built",
+                created_at=now,
+            )
+        )
+        session.add(
+            Page(
+                id=page_id,
+                url="https://example.test/legacy",
+                cleaned_text="Actor uses malware",
+                scrape_timestamp=now,
+                created_at=now,
+            )
+        )
+        session.add_all(
+            [
+                Entity(
+                    id=source_id,
+                    page_id=page_id,
+                    investigation_id=investigation_id,
+                    entity_type="THREAT_ACTOR_HANDLE",
+                    value="actor-one",
+                    canonical_value="actor-one",
+                    confidence=0.95,
+                    first_seen=now,
+                    last_seen=now,
+                    created_at=now,
+                ),
+                Entity(
+                    id=target_id,
+                    page_id=page_id,
+                    investigation_id=investigation_id,
+                    entity_type="MALWARE_FAMILY",
+                    value="malware-one",
+                    canonical_value="malware-one",
+                    confidence=0.95,
+                    first_seen=now,
+                    last_seen=now,
+                    created_at=now,
+                ),
+            ]
+        )
+        session.add(
+            EntityRelationship(
+                entity_a_id=source_id,
+                entity_b_id=target_id,
+                relationship_type="USES",
+                confidence=0.9,
+                investigation_id=investigation_id,
+                source_page_id=None,
+                first_seen=now,
+            )
+        )
+        session.commit()
+
+        result = run_backfill(session, investigation_id)
+        session.commit()
+
+        relationship = session.query(EntityRelationship).one()
+        investigation = session.get(Investigation, investigation_id)
+
+        assert result["typed_relationship_pages"]["resolved"] == 1
+        assert relationship.source_page_id == page_id
+        assert investigation.metadata_json["communities"]
+        assert investigation.metadata_json["community_count"] == 1
+
+
+def test_backfill_does_not_assign_invalid_typed_relationship_page(db_engine):
+    from db.models import Entity, EntityRelationship, Investigation, Page
+    from db.session import get_session
+    from voidaccess.backfill import backfill_typed_relationship_page_ids
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    investigation_id = uuid.uuid4()
+    page_id = uuid.uuid4()
+    source_id, target_id = uuid.uuid4(), uuid.uuid4()
+
+    with get_session() as session:
+        session.add(
+            Investigation(
+                id=investigation_id,
+                query="invalid legacy relationship",
+                status="completed",
+                created_at=now,
+            )
+        )
+        session.add(Page(id=page_id, url="https://example.test/invalid", scrape_timestamp=now, created_at=now))
+        session.add_all(
+            [
+                Entity(id=source_id, page_id=page_id, investigation_id=investigation_id,
+                       entity_type="ORGANIZATION_NAME", value="org", canonical_value="org"),
+                Entity(id=target_id, page_id=page_id, investigation_id=investigation_id,
+                       entity_type="IP_ADDRESS", value="192.0.2.1", canonical_value="192.0.2.1"),
+                EntityRelationship(entity_a_id=source_id, entity_b_id=target_id,
+                                   relationship_type="TARGETS", investigation_id=investigation_id,
+                                   first_seen=now),
+            ]
+        )
+        session.commit()
+        result = backfill_typed_relationship_page_ids(session, investigation_id)
+        assert result == {"scanned": 1, "resolved": 0, "unresolved": 1}

voidaccess-2.0.3/voidaccess/backfill.py

@@ -0,0 +1,251 @@
+"""One-time repairs for legacy investigation data.
+
+This module deliberately does not change the schema.  It repairs two pieces of
+derived data that were introduced after some investigations had already been
+stored: the page provenance on typed relationship edges and the persisted
+community partition used by the HTTP graph endpoint.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import uuid
+from collections import defaultdict
+from typing import Any, Optional
+
+from sqlalchemy.orm import joinedload
+
+from extractor.relationship_extract import _LLM_REL_VOCAB, _is_compatible_relationship
+from graph.builder import build_graph_from_db, detect_communities
+
+logger = logging.getLogger(__name__)
+
+
+def _as_uuid(value: Any) -> Optional[uuid.UUID]:
+    if isinstance(value, uuid.UUID):
+        return value
+    try:
+        return uuid.UUID(str(value)) if value else None
+    except (TypeError, ValueError, AttributeError):
+        return None
+
+
+def _metadata_dict(value: Any) -> dict[str, Any]:
+    if isinstance(value, dict):
+        return dict(value)
+    if isinstance(value, str) and value:
+        try:
+            decoded = json.loads(value)
+        except (TypeError, ValueError):
+            return {}
+        return dict(decoded) if isinstance(decoded, dict) else {}
+    return {}
+
+
+def _validated_entities(session, investigation_id: uuid.UUID):
+    """Return the current entity set, including cross-investigation links."""
+    from db.models import Entity, InvestigationEntityLink
+
+    return (
+        session.query(Entity)
+        .outerjoin(
+            InvestigationEntityLink,
+            InvestigationEntityLink.entity_id == Entity.id,
+        )
+        .filter(
+            (Entity.investigation_id == investigation_id)
+            | (InvestigationEntityLink.investigation_id == investigation_id)
+        )
+        .options(joinedload(Entity.page))
+        .distinct()
+        .all()
+    )
+
+
+def _resolve_source_page_id(
+    relationship,
+    entity_a,
+    entity_b,
+    validated_entities: list,
+) -> Optional[uuid.UUID]:
+    """Resolve provenance using the live pipeline's validated entity set.
+
+    Typed relationship direction is meaningful: the live pipeline persists the
+    page on which the claim was extracted.  For a legacy row without that
+    field, the source endpoint's page is the strongest deterministic evidence;
+    the target page and a page containing both endpoints are conservative
+    fallbacks.  No page outside the validated investigation set is accepted.
+    """
+    if not _is_compatible_relationship(
+        str(relationship.relationship_type or ""),
+        {"type": entity_a.entity_type},
+        {"type": entity_b.entity_type},
+    ):
+        return None
+
+    valid_page_ids = {
+        ent.page_id for ent in validated_entities if ent.page_id is not None
+    }
+    if not valid_page_ids:
+        return None
+
+    page_entity_ids: dict[uuid.UUID, set[uuid.UUID]] = defaultdict(set)
+    for ent in validated_entities:
+        if ent.page_id is not None:
+            page_entity_ids[ent.page_id].add(ent.id)
+
+    # The source endpoint is the direct equivalent of the page_id attached by
+    # relationship_extract.extract_relationships_from_results().
+    candidates = [
+        entity_a.page_id,
+        entity_b.page_id,
+    ]
+    for page_id, entity_ids in page_entity_ids.items():
+        if entity_a.id in entity_ids and entity_b.id in entity_ids:
+            candidates.append(page_id)
+    for page_id, entity_ids in page_entity_ids.items():
+        if entity_a.id in entity_ids:
+            candidates.append(page_id)
+
+    seen: set[uuid.UUID] = set()
+    for page_id in candidates:
+        if page_id is None or page_id in seen:
+            continue
+        seen.add(page_id)
+        if page_id in valid_page_ids:
+            return page_id
+    return None
+
+
+def backfill_typed_relationship_page_ids(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, int]:
+    """Populate missing ``source_page_id`` values on validated typed edges."""
+    from db.models import Entity, EntityRelationship
+
+    typed_types = set(_LLM_REL_VOCAB.values())
+    query = session.query(EntityRelationship).filter(
+        EntityRelationship.source_page_id.is_(None),
+        EntityRelationship.relationship_type.in_(typed_types),
+    )
+    if investigation_id is not None:
+        query = query.filter(EntityRelationship.investigation_id == investigation_id)
+
+    rows = query.options(
+        joinedload(EntityRelationship.entity_a).joinedload(Entity.page),
+        joinedload(EntityRelationship.entity_b).joinedload(Entity.page),
+    ).all()
+    entities_by_investigation: dict[uuid.UUID, list] = {}
+    scanned = resolved = unresolved = 0
+
+    for relationship in rows:
+        scanned += 1
+        entity_a = relationship.entity_a
+        entity_b = relationship.entity_b
+        target_investigation = (
+            _as_uuid(relationship.investigation_id)
+            or _as_uuid(getattr(entity_a, "investigation_id", None))
+            or _as_uuid(getattr(entity_b, "investigation_id", None))
+        )
+        if target_investigation is None:
+            unresolved += 1
+            continue
+        if target_investigation not in entities_by_investigation:
+            entities_by_investigation[target_investigation] = _validated_entities(
+                session, target_investigation
+            )
+        page_id = _resolve_source_page_id(
+            relationship,
+            entity_a,
+            entity_b,
+            entities_by_investigation[target_investigation],
+        )
+        if page_id is None:
+            unresolved += 1
+            continue
+        relationship.source_page_id = page_id
+        resolved += 1
+
+    if resolved:
+        session.flush()
+    return {"scanned": scanned, "resolved": resolved, "unresolved": unresolved}
+
+
+def backfill_community_partitions(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, int]:
+    """Persist detector output for completed investigations lacking a partition."""
+    from db.models import Investigation
+
+    query = session.query(Investigation).filter(Investigation.status == "completed")
+    if investigation_id is not None:
+        query = query.filter(Investigation.id == investigation_id)
+
+    scanned = persisted = skipped = 0
+    for investigation in query.order_by(Investigation.created_at).all():
+        scanned += 1
+        metadata = _metadata_dict(investigation.metadata_json)
+        existing = metadata.get("communities")
+        if isinstance(existing, dict) and existing:
+            skipped += 1
+            continue
+
+        graph = build_graph_from_db(investigation_id=investigation.id)
+        partition = {
+            str(node_id): int(community_id)
+            for node_id, community_id in detect_communities(graph).items()
+        }
+        metadata["communities"] = partition
+        metadata["community_count"] = len(set(partition.values())) if partition else 0
+        investigation.metadata_json = metadata
+        session.flush()
+        persisted += 1
+
+    return {"scanned": scanned, "persisted": persisted, "skipped": skipped}
+
+
+def run_backfill(
+    session,
+    investigation_id: Optional[uuid.UUID] = None,
+) -> dict[str, dict[str, int]]:
+    """Run both repairs in one transaction."""
+    typed = backfill_typed_relationship_page_ids(session, investigation_id)
+    communities = backfill_community_partitions(session, investigation_id)
+    return {"typed_relationship_pages": typed, "community_partitions": communities}
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--db-path", help="SQLite database path (defaults to VOIDACCESS_DB_PATH)")
+    parser.add_argument("--investigation-id", help="Limit the repair to one investigation")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args(argv)
+
+    if args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    db_path = args.db_path or os.getenv("VOIDACCESS_DB_PATH")
+    if db_path:
+        db_path = os.path.abspath(os.path.expanduser(db_path))
+        os.environ["VOIDACCESS_DB_PATH"] = db_path
+        os.environ["DATABASE_URL"] = f"sqlite:///{db_path.replace(os.sep, '/') }"
+
+    target_id = _as_uuid(args.investigation_id)
+    if args.investigation_id and target_id is None:
+        parser.error("--investigation-id must be a UUID")
+
+    from db.session import get_session
+
+    with get_session() as session:
+        result = run_backfill(session, target_id)
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{voidaccess-2.0.2 → voidaccess-2.0.3}/voidaccess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 2.0.2
+Version: 2.0.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-2.0.2 → voidaccess-2.0.3}/voidaccess.egg-info/SOURCES.txt

@@ -186,6 +186,7 @@ tests/test_i18n.py
 tests/test_infostealer.py
 tests/test_ioc_package_export.py
 tests/test_ip_reputation.py
+tests/test_legacy_backfill.py
 tests/test_llm.py
 tests/test_llm_utils.py
 tests/test_model_singleton.py
@@ -232,6 +233,7 @@ vector/model_singleton.py
 vector/search.py
 vector/store.py
 voidaccess/__init__.py
+voidaccess/backfill.py
 voidaccess/config.py
 voidaccess/llm.py
 voidaccess/llm_utils.py

{voidaccess-2.0.2 → voidaccess-2.0.3}/voidaccess_cli/__init__.py

@@ -1,3 +1,3 @@
 """voidaccess CLI — dark-web OSINT command-line interface."""
 
-__version__ = "2.0.2"
+__version__ = "2.0.3"

{voidaccess-2.0.2 → voidaccess-2.0.3}/voidaccess_cli/config.py

@@ -20,6 +20,7 @@ _WARNED_CONFIG_KEYS: set[str] = set()
 
 CLI_HOME = Path(os.path.expanduser("~/.voidaccess"))
 CONFIG_PATH = CLI_HOME / "config.json"
+ENV_PATH = Path.cwd() / ".env"
 DB_PATH = CLI_HOME / "investigations.db"
 DEFAULT_OUTPUT_DIR = CLI_HOME / "results"
 
@@ -248,6 +249,19 @@ def _lock_file(path: Path) -> None:
         pass
 
 
+def lock_env_file() -> None:
+    """Apply the same owner-only protection to the project ``.env`` file.
+
+    ``voidaccess configure`` stores its CLI settings separately in
+    ``~/.voidaccess/config.json``, but a self-hosted checkout may also have a
+    project-level ``.env`` containing the API/database credentials consumed by
+    the runtime.  Lock it whenever configuration is saved, while keeping the
+    operation a no-op for CLI-only installs that have no ``.env``.
+    """
+    if ENV_PATH.is_file():
+        _lock_file(ENV_PATH)
+
+
 def save_config(config: dict[str, Any]) -> None:
     features = config.setdefault("features", {})
     # Keep the legacy aliases readable for older clients.  When a caller
@@ -271,6 +285,7 @@ def save_config(config: dict[str, Any]) -> None:
         encoding="utf-8",
     )
     _lock_file(CONFIG_PATH)
+    lock_env_file()
 
 
 def is_configured() -> bool: