voidaccess 1.6.2__tar.gz → 1.6.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voidaccess-1.6.2/voidaccess.egg-info → voidaccess-1.6.3}/PKG-INFO +7 -4
- {voidaccess-1.6.2 → voidaccess-1.6.3}/README.md +4 -3
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/ioc_package.py +1 -1
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/llm_extract.py +2 -27
- {voidaccess-1.6.2 → voidaccess-1.6.3}/pyproject.toml +4 -2
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/proxy_client.py +4 -4
- {voidaccess-1.6.2 → voidaccess-1.6.3/voidaccess.egg-info}/PKG-INFO +7 -4
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess.egg-info/requires.txt +2 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/__init__.py +1 -1
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/configure.py +14 -9
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/config.py +47 -6
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/main.py +1 -1
- {voidaccess-1.6.2 → voidaccess-1.6.3}/LICENSE +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/analysis/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/analysis/opsec.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/analysis/patterns.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/analysis/temporal.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/auth.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/main.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/actors.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/admin.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/auth.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/entities.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/investigations.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/monitors.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/search.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/api/routes/settings.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/auth/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/auth/token_blacklist.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/config.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/crawler/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/crawler/dedup.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/crawler/frontier.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/crawler/spider.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/crawler/utils.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/env.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0001_initial_schema.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0002_add_missing_tables.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0005_add_extraction_method.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0008_add_users_table.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0013_add_graph_status.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0015_add_progress_fields.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/models.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/queries.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/search_engine_stats.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/db/session.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/misp.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/sigma.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/snort_export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/stix.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/export/yara_export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/ner.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/normalizer.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/pipeline.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/extractor/regex_patterns.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/fingerprint/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/fingerprint/profiler.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/fingerprint/stylometry.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/builder.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/model.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/queries.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/graph/visualize.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/i18n/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/i18n/detect.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/i18n/query_expand.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/i18n/translate.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/_db.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/alerts.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/config.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/diff.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/jobs.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/monitor/scheduler.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/scraper/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/scraper/scrape.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/scraper/scrape_js.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/search/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/search/circuit_breaker.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/search/query_builder.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/search/search.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/setup.cfg +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/actor_profiles.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/blockchain.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/cache.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/cisa.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/data/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/data/onion_seeds.json +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/dns_enrichment.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/domain_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/email_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/engines.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/enrichment.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/github_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/gitlab_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/hash_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/historical_intel.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/ip_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/paste_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/pastes.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/rss_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/seed_manager.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/seeds.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/shodan.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/telegram.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/sources/virustotal.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_analysis_opsec.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_analysis_stylometry.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_analysis_temporal.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_api.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_api_monitors.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_blockchain.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_cli_proxy_config.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_config.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_crawler.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_db.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_dns_enrichment.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_domain_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_email_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_filter_e2e_manual.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_filter_parser_manual.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_fingerprint.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_github_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_gitlab_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_graph.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_hash_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_i18n.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_ip_reputation.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_llm.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_llm_utils.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_model_singleton.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_monitor.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_pagination.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_paste_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_proxy_client.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_regex_patterns.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_rss_scraper.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_scrape_js.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_settings.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_snort_export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_sources.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_sources_enrichment_new.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_vector.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/tests/test_yara_export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/async_utils.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/content_safety.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/defang.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/encryption.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/enrichment_cache.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/ioc_freshness.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/utils/user_keys.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/vector/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/vector/embedder.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/vector/model_singleton.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/vector/search.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/vector/store.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess/config.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess/llm.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess/llm_utils.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess.egg-info/SOURCES.txt +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess.egg-info/dependency_links.txt +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess.egg-info/entry_points.txt +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess.egg-info/top_level.txt +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/adapters/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/adapters/sqlite.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/browser.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/__init__.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/actors.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/enrich.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/export.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/investigate.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/commands/show.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/display.py +0 -0
- {voidaccess-1.6.2 → voidaccess-1.6.3}/voidaccess_cli/tor_detect.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: voidaccess
|
|
3
|
-
Version: 1.6.
|
|
3
|
+
Version: 1.6.3
|
|
4
4
|
Summary: Dark web OSINT CLI — automated threat intelligence from query to report
|
|
5
5
|
Author: VoidAccess
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,7 @@ License-File: LICENSE
|
|
|
19
19
|
Requires-Dist: typer>=0.12
|
|
20
20
|
Requires-Dist: rich>=13
|
|
21
21
|
Requires-Dist: textual>=0.60
|
|
22
|
+
Requires-Dist: click>=8.0
|
|
22
23
|
Requires-Dist: aiohttp>=3.9
|
|
23
24
|
Requires-Dist: aiohttp-socks>=0.8
|
|
24
25
|
Requires-Dist: sqlalchemy>=2.0
|
|
@@ -39,6 +40,7 @@ Requires-Dist: trafilatura>=1.6
|
|
|
39
40
|
Requires-Dist: requests>=2.31
|
|
40
41
|
Requires-Dist: python-socks>=2.4
|
|
41
42
|
Requires-Dist: tldextract>=5.1
|
|
43
|
+
Requires-Dist: stix2>=3.0.2
|
|
42
44
|
Provides-Extra: dev
|
|
43
45
|
Requires-Dist: pytest; extra == "dev"
|
|
44
46
|
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
@@ -65,8 +67,9 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
65
67
|
|
|
66
68
|
---
|
|
67
69
|
|
|
68
|
-
## What's New in v1.6.
|
|
70
|
+
## What's New in v1.6.3
|
|
69
71
|
|
|
72
|
+
- **Release candidate alignment** — release metadata, package metadata, and docs now all report `v1.6.3`.
|
|
70
73
|
- **Optional clearnet ScrapingAnt integration** — paste sites and RSS feeds can now be routed through ScrapingAnt. Affects clearnet scraping only; Tor, `.onion`, GitHub, and GitLab traffic are never affected.
|
|
71
74
|
- **Three independent products** — Web Scraping API, Residential Proxy transport, and Datacenter Proxy transport are documented separately so the API key is not conflated with the proxy credentials.
|
|
72
75
|
- **One-shot CLI flags** — `voidaccess investigate --use-scraping-api` and `voidaccess investigate --use-proxies` select the REST API or proxy transport for a single run.
|
|
@@ -76,7 +79,7 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
76
79
|
|
|
77
80
|
---
|
|
78
81
|
|
|
79
|
-
## What's New in v1.
|
|
82
|
+
## What's New in v1.6.3
|
|
80
83
|
|
|
81
84
|
- 37 new entity types across crypto, credentials, messaging, and network/forensic indicators.
|
|
82
85
|
- YARA, Snort, Suricata, and IOC package ZIP exports.
|
|
@@ -270,7 +273,7 @@ Export formats:
|
|
|
270
273
|
|
|
271
274
|
## Actor Intelligence
|
|
272
275
|
|
|
273
|
-
VoidAccess v1.
|
|
276
|
+
VoidAccess v1.6.3 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
|
|
274
277
|
|
|
275
278
|
Cross-alias resolution scores five signals: shared infrastructure, shared PGP, string similarity, temporal co-activity, and co-investigation. Use `voidaccess actors` to list profiles, `voidaccess actor <handle>` for the full profile, `voidaccess actor <handle> --timeline` for chronology, and `voidaccess actor <handle> --note "text"` for analyst notes.
|
|
276
279
|
|
|
@@ -19,8 +19,9 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
19
19
|
|
|
20
20
|
---
|
|
21
21
|
|
|
22
|
-
## What's New in v1.6.
|
|
22
|
+
## What's New in v1.6.3
|
|
23
23
|
|
|
24
|
+
- **Release candidate alignment** — release metadata, package metadata, and docs now all report `v1.6.3`.
|
|
24
25
|
- **Optional clearnet ScrapingAnt integration** — paste sites and RSS feeds can now be routed through ScrapingAnt. Affects clearnet scraping only; Tor, `.onion`, GitHub, and GitLab traffic are never affected.
|
|
25
26
|
- **Three independent products** — Web Scraping API, Residential Proxy transport, and Datacenter Proxy transport are documented separately so the API key is not conflated with the proxy credentials.
|
|
26
27
|
- **One-shot CLI flags** — `voidaccess investigate --use-scraping-api` and `voidaccess investigate --use-proxies` select the REST API or proxy transport for a single run.
|
|
@@ -30,7 +31,7 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
30
31
|
|
|
31
32
|
---
|
|
32
33
|
|
|
33
|
-
## What's New in v1.
|
|
34
|
+
## What's New in v1.6.3
|
|
34
35
|
|
|
35
36
|
- 37 new entity types across crypto, credentials, messaging, and network/forensic indicators.
|
|
36
37
|
- YARA, Snort, Suricata, and IOC package ZIP exports.
|
|
@@ -224,7 +225,7 @@ Export formats:
|
|
|
224
225
|
|
|
225
226
|
## Actor Intelligence
|
|
226
227
|
|
|
227
|
-
VoidAccess v1.
|
|
228
|
+
VoidAccess v1.6.3 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
|
|
228
229
|
|
|
229
230
|
Cross-alias resolution scores five signals: shared infrastructure, shared PGP, string similarity, temporal co-activity, and co-investigation. Use `voidaccess actors` to list profiles, `voidaccess actor <handle>` for the full profile, `voidaccess actor <handle> --timeline` for chronology, and `voidaccess actor <handle> --note "text"` for analyst notes.
|
|
230
231
|
|
|
@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
|
|
|
54
54
|
# ---------------------------------------------------------------------------
|
|
55
55
|
|
|
56
56
|
PACKAGE_FORMAT = "voidaccess-ioc-v1"
|
|
57
|
-
PACKAGE_VERSION = "1.
|
|
57
|
+
PACKAGE_VERSION = "1.6.3"
|
|
58
58
|
SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
|
|
59
59
|
|
|
60
60
|
# Map extractor upper-case entity types to the per-file bucket they belong to.
|
|
@@ -404,34 +404,9 @@ async def _extract_chunk(chunk: str, llm) -> str:
|
|
|
404
404
|
generate_summary) keep their streaming output unchanged.
|
|
405
405
|
"""
|
|
406
406
|
try:
|
|
407
|
-
from langchain_core.callbacks import BaseCallbackHandler
|
|
408
|
-
|
|
409
|
-
class _SilentHandler(BaseCallbackHandler):
|
|
410
|
-
"""No-op callback that suppresses the streaming handler's stdout
|
|
411
|
-
prints for this single extraction call. Every hook is a
|
|
412
|
-
no-op; we only care about preventing the parent's stdout
|
|
413
|
-
noise — we still want ``ainvoke`` to return the full response
|
|
414
|
-
content as before, just without the per-token terminal spam.
|
|
415
|
-
"""
|
|
416
|
-
|
|
417
|
-
def on_llm_new_token(self, token: str, **kwargs) -> None: # noqa: D401
|
|
418
|
-
return None
|
|
419
|
-
|
|
420
|
-
def on_llm_end(self, response, **kwargs) -> None: # noqa: D401
|
|
421
|
-
return None
|
|
422
|
-
|
|
423
|
-
def on_llm_error(self, error, **kwargs) -> None: # noqa: D401
|
|
424
|
-
return None
|
|
425
|
-
|
|
426
407
|
prompt = _PROMPT_TEMPLATE.format(chunk=chunk)
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
# other call sites (refine_query, filter_results, summary) are
|
|
430
|
-
# not affected.
|
|
431
|
-
response = await llm.ainvoke(
|
|
432
|
-
prompt,
|
|
433
|
-
config={"callbacks": [_SilentHandler()]},
|
|
434
|
-
)
|
|
408
|
+
silent_llm = llm.bind(streaming=False, callbacks=[])
|
|
409
|
+
response = await silent_llm.ainvoke(prompt)
|
|
435
410
|
content = response.content if hasattr(response, "content") else str(response)
|
|
436
411
|
return content.strip()
|
|
437
412
|
except Exception as exc:
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "voidaccess"
|
|
7
|
-
version = "1.6.
|
|
7
|
+
version = "1.6.3"
|
|
8
8
|
description = "Dark web OSINT CLI — automated threat intelligence from query to report"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -25,6 +25,7 @@ dependencies = [
|
|
|
25
25
|
"typer>=0.12",
|
|
26
26
|
"rich>=13",
|
|
27
27
|
"textual>=0.60",
|
|
28
|
+
"click>=8.0",
|
|
28
29
|
"aiohttp>=3.9",
|
|
29
30
|
"aiohttp-socks>=0.8",
|
|
30
31
|
"sqlalchemy>=2.0",
|
|
@@ -45,6 +46,7 @@ dependencies = [
|
|
|
45
46
|
"requests>=2.31",
|
|
46
47
|
"python-socks>=2.4",
|
|
47
48
|
"tldextract>=5.1",
|
|
49
|
+
"stix2>=3.0.2",
|
|
48
50
|
]
|
|
49
51
|
|
|
50
52
|
[project.optional-dependencies]
|
|
@@ -92,7 +94,7 @@ exclude = [
|
|
|
92
94
|
"__pycache__*",
|
|
93
95
|
]
|
|
94
96
|
|
|
95
|
-
# v1.6.
|
|
97
|
+
# v1.6.3 — ship data/onion_seeds.json inside the installed package.
|
|
96
98
|
# Without this entry, ``pip install voidaccess`` produces a package
|
|
97
99
|
# directory that does NOT contain the curated seed list — every
|
|
98
100
|
# CLI investigation silently gets zero seed injection, which is the
|
|
@@ -6,7 +6,7 @@ single-axis dispatch on clearnet HTTP fetches:
|
|
|
6
6
|
|
|
7
7
|
Transport: How requests reach ScrapingAnt's backend
|
|
8
8
|
--------- --------------------------------------------------------
|
|
9
|
-
direct Local aiohttp.ClientSession, no proxy (
|
|
9
|
+
direct Local aiohttp.ClientSession, no proxy (pre-1.6.2 behavior)
|
|
10
10
|
api POST to https://api.scrapingant.com/v2/general (REST)
|
|
11
11
|
proxy HTTP CONNECT through the configured ScrapingAnt proxy endpoint (proxy transport)
|
|
12
12
|
|
|
@@ -60,7 +60,7 @@ Design:
|
|
|
60
60
|
This is a routing invariant, not a proxy concern.
|
|
61
61
|
- Single transport per request, picked by config:
|
|
62
62
|
- If VOIDACCESS_USE_PROXY=true (and SCRAPINGANT_API_KEY set) → proxy
|
|
63
|
-
- Else if VOIDACCESS_USE_PROXIES=true (legacy
|
|
63
|
+
- Else if VOIDACCESS_USE_PROXIES=true (legacy pre-1.6.2,
|
|
64
64
|
and SCRAPINGANT_API_KEY set) → api
|
|
65
65
|
- Else → direct
|
|
66
66
|
If BOTH VOIDACCESS_USE_PROXY and VOIDACCESS_USE_PROXIES are set,
|
|
@@ -226,7 +226,7 @@ def is_api_transport_enabled() -> bool:
|
|
|
226
226
|
"""Return True when the REST API transport should be used.
|
|
227
227
|
|
|
228
228
|
Triggered by:
|
|
229
|
-
- VOIDACCESS_USE_PROXIES=true (legacy
|
|
229
|
+
- VOIDACCESS_USE_PROXIES=true (legacy pre-1.6.2 alias — kept for
|
|
230
230
|
backward compatibility with Phase 1 deployments)
|
|
231
231
|
|
|
232
232
|
Requires SCRAPINGANT_API_KEY to be set and non-empty.
|
|
@@ -378,7 +378,7 @@ async def clearnet_fetch(
|
|
|
378
378
|
|
|
379
379
|
Picks ONE transport based on env-var configuration:
|
|
380
380
|
- direct (default, when no ScrapingAnt env vars are set)
|
|
381
|
-
- api (VOIDACCESS_USE_PROXIES=true; legacy
|
|
381
|
+
- api (VOIDACCESS_USE_PROXIES=true; legacy pre-1.6.2 alias)
|
|
382
382
|
- proxy (VOIDACCESS_USE_PROXY=true; proxy transport per docs)
|
|
383
383
|
|
|
384
384
|
Per https://docs.scrapingant.com/proxy-mode: "The proxy transport is a
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: voidaccess
|
|
3
|
-
Version: 1.6.
|
|
3
|
+
Version: 1.6.3
|
|
4
4
|
Summary: Dark web OSINT CLI — automated threat intelligence from query to report
|
|
5
5
|
Author: VoidAccess
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,7 @@ License-File: LICENSE
|
|
|
19
19
|
Requires-Dist: typer>=0.12
|
|
20
20
|
Requires-Dist: rich>=13
|
|
21
21
|
Requires-Dist: textual>=0.60
|
|
22
|
+
Requires-Dist: click>=8.0
|
|
22
23
|
Requires-Dist: aiohttp>=3.9
|
|
23
24
|
Requires-Dist: aiohttp-socks>=0.8
|
|
24
25
|
Requires-Dist: sqlalchemy>=2.0
|
|
@@ -39,6 +40,7 @@ Requires-Dist: trafilatura>=1.6
|
|
|
39
40
|
Requires-Dist: requests>=2.31
|
|
40
41
|
Requires-Dist: python-socks>=2.4
|
|
41
42
|
Requires-Dist: tldextract>=5.1
|
|
43
|
+
Requires-Dist: stix2>=3.0.2
|
|
42
44
|
Provides-Extra: dev
|
|
43
45
|
Requires-Dist: pytest; extra == "dev"
|
|
44
46
|
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
@@ -65,8 +67,9 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
65
67
|
|
|
66
68
|
---
|
|
67
69
|
|
|
68
|
-
## What's New in v1.6.
|
|
70
|
+
## What's New in v1.6.3
|
|
69
71
|
|
|
72
|
+
- **Release candidate alignment** — release metadata, package metadata, and docs now all report `v1.6.3`.
|
|
70
73
|
- **Optional clearnet ScrapingAnt integration** — paste sites and RSS feeds can now be routed through ScrapingAnt. Affects clearnet scraping only; Tor, `.onion`, GitHub, and GitLab traffic are never affected.
|
|
71
74
|
- **Three independent products** — Web Scraping API, Residential Proxy transport, and Datacenter Proxy transport are documented separately so the API key is not conflated with the proxy credentials.
|
|
72
75
|
- **One-shot CLI flags** — `voidaccess investigate --use-scraping-api` and `voidaccess investigate --use-proxies` select the REST API or proxy transport for a single run.
|
|
@@ -76,7 +79,7 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
|
|
|
76
79
|
|
|
77
80
|
---
|
|
78
81
|
|
|
79
|
-
## What's New in v1.
|
|
82
|
+
## What's New in v1.6.3
|
|
80
83
|
|
|
81
84
|
- 37 new entity types across crypto, credentials, messaging, and network/forensic indicators.
|
|
82
85
|
- YARA, Snort, Suricata, and IOC package ZIP exports.
|
|
@@ -270,7 +273,7 @@ Export formats:
|
|
|
270
273
|
|
|
271
274
|
## Actor Intelligence
|
|
272
275
|
|
|
273
|
-
VoidAccess v1.
|
|
276
|
+
VoidAccess v1.6.3 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
|
|
274
277
|
|
|
275
278
|
Cross-alias resolution scores five signals: shared infrastructure, shared PGP, string similarity, temporal co-activity, and co-investigation. Use `voidaccess actors` to list profiles, `voidaccess actor <handle>` for the full profile, `voidaccess actor <handle> --timeline` for chronology, and `voidaccess actor <handle> --note "text"` for analyst notes.
|
|
276
279
|
|
|
@@ -242,23 +242,27 @@ def _prompt_scrapingant(cfg: dict) -> None:
|
|
|
242
242
|
# both means the chokepoint picks proxy (logged at runtime with a
|
|
243
243
|
# one-shot info message).
|
|
244
244
|
if new_key.strip():
|
|
245
|
-
current_api = bool(cfg.get("features", {}).get("use_proxies", False))
|
|
245
|
+
current_api = bool(cfg.get("features", {}).get("rest_api_transport_enabled", cfg.get("features", {}).get("use_proxies", False)))
|
|
246
246
|
if Confirm.ask(
|
|
247
247
|
" Enable REST API transport (ScrapingAnt Web Scraping API) for clearnet scrapes?",
|
|
248
248
|
default=current_api,
|
|
249
249
|
):
|
|
250
|
+
cfg.setdefault("features", {})["rest_api_transport_enabled"] = True
|
|
250
251
|
cfg.setdefault("features", {})["use_proxies"] = True
|
|
251
252
|
else:
|
|
253
|
+
cfg.setdefault("features", {})["rest_api_transport_enabled"] = False
|
|
252
254
|
cfg.setdefault("features", {})["use_proxies"] = False
|
|
253
255
|
|
|
254
|
-
current_proxy = bool(cfg.get("features", {}).get("use_proxy", False))
|
|
256
|
+
current_proxy = bool(cfg.get("features", {}).get("residential_proxy_enabled", cfg.get("features", {}).get("use_proxy", False)))
|
|
255
257
|
if Confirm.ask(
|
|
256
258
|
f" Enable proxy transport (HTTP CONNECT through the configured ScrapingAnt proxy endpoint, "
|
|
257
259
|
f"{new_type} pool) for clearnet scrapes?",
|
|
258
260
|
default=current_proxy,
|
|
259
261
|
):
|
|
262
|
+
cfg.setdefault("features", {})["residential_proxy_enabled"] = True
|
|
260
263
|
cfg.setdefault("features", {})["use_proxy"] = True
|
|
261
264
|
else:
|
|
265
|
+
cfg.setdefault("features", {})["residential_proxy_enabled"] = False
|
|
262
266
|
cfg.setdefault("features", {})["use_proxy"] = False
|
|
263
267
|
|
|
264
268
|
|
|
@@ -334,7 +338,7 @@ def configure_proxy(
|
|
|
334
338
|
enable: Optional[bool] = typer.Option(
|
|
335
339
|
None,
|
|
336
340
|
"--enable/--disable",
|
|
337
|
-
help="Enable or disable the REST API transport (legacy
|
|
341
|
+
help="Enable or disable the REST API transport (legacy pre-1.6.2 toggle).",
|
|
338
342
|
),
|
|
339
343
|
enable_proxy: Optional[bool] = typer.Option(
|
|
340
344
|
None,
|
|
@@ -350,7 +354,7 @@ def configure_proxy(
|
|
|
350
354
|
"""Configure the optional clearnet proxy (ScrapingAnt).
|
|
351
355
|
|
|
352
356
|
With no flags, runs the interactive prompt. With --enable / --disable,
|
|
353
|
-
sets the REST API transport toggle non-interactively (legacy
|
|
357
|
+
sets the REST API transport toggle non-interactively (legacy pre-1.6.2
|
|
354
358
|
flag). With --enable-proxy / --disable-proxy, sets the proxy transport
|
|
355
359
|
transport toggle (new in v1.6.2 — requires SCRAPINGANT_API_KEY to
|
|
356
360
|
actually activate).
|
|
@@ -364,7 +368,6 @@ def configure_proxy(
|
|
|
364
368
|
Clearnet scraping only — Tor and .onion traffic are never affected.
|
|
365
369
|
"""
|
|
366
370
|
cfg = cli_config.load_config()
|
|
367
|
-
|
|
368
371
|
# --show: display current state with the key masked, and exit.
|
|
369
372
|
if show and enable is None and enable_proxy is None:
|
|
370
373
|
key = cfg.get("enrichment_keys", {}).get("SCRAPINGANT_API_KEY", "")
|
|
@@ -373,8 +376,8 @@ def configure_proxy(
|
|
|
373
376
|
proxy_type = (
|
|
374
377
|
cfg.get("enrichment_keys", {}).get("SCRAPINGANT_PROXY_TYPE", "") or "—"
|
|
375
378
|
)
|
|
376
|
-
api_transport = bool(cfg.get("features", {}).get("use_proxies", False))
|
|
377
|
-
proxy_transport = bool(cfg.get("features", {}).get("use_proxy", False))
|
|
379
|
+
api_transport = bool(cfg.get("features", {}).get("rest_api_transport_enabled", cfg.get("features", {}).get("use_proxies", False)))
|
|
380
|
+
proxy_transport = bool(cfg.get("features", {}).get("residential_proxy_enabled", cfg.get("features", {}).get("use_proxy", False)))
|
|
378
381
|
|
|
379
382
|
masked_key = (
|
|
380
383
|
f"{key[:4]}…{key[-4:]}" if len(key) > 8 else ("set" if key else "—")
|
|
@@ -390,8 +393,8 @@ def configure_proxy(
|
|
|
390
393
|
console.print(f" Pool type : {proxy_type}")
|
|
391
394
|
api_label = "[green]enabled[/green]" if api_transport else "[red]disabled[/red]"
|
|
392
395
|
proxy_label = "[green]enabled[/green]" if proxy_transport else "[red]disabled[/red]"
|
|
393
|
-
console.print(f" API transport
|
|
394
|
-
console.print(f"
|
|
396
|
+
console.print(f" REST API transport : {api_label}")
|
|
397
|
+
console.print(f" Residential proxy : {proxy_label}")
|
|
395
398
|
console.print(" Scope : paste + RSS only, never Tor")
|
|
396
399
|
if proxy_transport and (not proxy_username or not proxy_password):
|
|
397
400
|
console.print(
|
|
@@ -409,6 +412,7 @@ def configure_proxy(
|
|
|
409
412
|
return
|
|
410
413
|
|
|
411
414
|
if enable is not None:
|
|
415
|
+
cfg.setdefault("features", {})["rest_api_transport_enabled"] = bool(enable)
|
|
412
416
|
cfg.setdefault("features", {})["use_proxies"] = bool(enable)
|
|
413
417
|
cli_config.save_config(cfg)
|
|
414
418
|
state = "enabled" if enable else "disabled"
|
|
@@ -422,6 +426,7 @@ def configure_proxy(
|
|
|
422
426
|
return
|
|
423
427
|
|
|
424
428
|
if enable_proxy is not None:
|
|
429
|
+
cfg.setdefault("features", {})["residential_proxy_enabled"] = bool(enable_proxy)
|
|
425
430
|
cfg.setdefault("features", {})["use_proxy"] = bool(enable_proxy)
|
|
426
431
|
cli_config.save_config(cfg)
|
|
427
432
|
state = "enabled" if enable_proxy else "disabled"
|
|
@@ -11,9 +11,14 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
13
|
import os
|
|
14
|
+
import logging
|
|
15
|
+
import warnings
|
|
14
16
|
from pathlib import Path
|
|
15
17
|
from typing import Any, Optional
|
|
16
18
|
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
_DEPRECATION_NOTIFIED = False
|
|
21
|
+
|
|
17
22
|
CLI_HOME = Path(os.path.expanduser("~/.voidaccess"))
|
|
18
23
|
CONFIG_PATH = CLI_HOME / "config.json"
|
|
19
24
|
DB_PATH = CLI_HOME / "investigations.db"
|
|
@@ -82,7 +87,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
|
|
82
87
|
#
|
|
83
88
|
# use_proxies → VOIDACCESS_USE_PROXIES=true
|
|
84
89
|
# Selects the REST API transport.
|
|
85
|
-
# (legacy
|
|
90
|
+
# (legacy pre-1.6.2 alias; Phase 1 verified
|
|
86
91
|
# correct, must not be touched.)
|
|
87
92
|
#
|
|
88
93
|
# use_proxy → VOIDACCESS_USE_PROXY=true
|
|
@@ -99,6 +104,8 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
|
|
99
104
|
# emits a one-shot info log.
|
|
100
105
|
"use_proxies": False,
|
|
101
106
|
"use_proxy": False,
|
|
107
|
+
"rest_api_transport_enabled": False,
|
|
108
|
+
"residential_proxy_enabled": False,
|
|
102
109
|
},
|
|
103
110
|
"tor": {
|
|
104
111
|
"host": "127.0.0.1",
|
|
@@ -118,7 +125,7 @@ def load_config() -> dict[str, Any]:
|
|
|
118
125
|
if not CONFIG_PATH.exists():
|
|
119
126
|
return json.loads(json.dumps(DEFAULT_CONFIG))
|
|
120
127
|
try:
|
|
121
|
-
cfg = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
128
|
+
cfg = json.loads(CONFIG_PATH.read_text(encoding="utf-8-sig"))
|
|
122
129
|
except Exception:
|
|
123
130
|
return json.loads(json.dumps(DEFAULT_CONFIG))
|
|
124
131
|
# Merge with defaults so missing keys don't crash
|
|
@@ -127,12 +134,44 @@ def load_config() -> dict[str, Any]:
|
|
|
127
134
|
merged["tor"].update(cfg.get("tor", {}))
|
|
128
135
|
merged["enrichment_keys"].update(cfg.get("enrichment_keys", {}))
|
|
129
136
|
merged["features"].update(cfg.get("features", {}))
|
|
137
|
+
features = merged["features"]
|
|
138
|
+
global _DEPRECATION_NOTIFIED
|
|
139
|
+
deprecated_keys = []
|
|
140
|
+
if "rest_api_transport_enabled" not in cfg.get("features", {}) and "use_proxies" in cfg.get("features", {}):
|
|
141
|
+
features["rest_api_transport_enabled"] = bool(cfg["features"].get("use_proxies"))
|
|
142
|
+
deprecated_keys.append("use_proxies")
|
|
143
|
+
if "residential_proxy_enabled" not in cfg.get("features", {}) and "use_proxy" in cfg.get("features", {}):
|
|
144
|
+
features["residential_proxy_enabled"] = bool(cfg["features"].get("use_proxy"))
|
|
145
|
+
deprecated_keys.append("use_proxy")
|
|
146
|
+
if deprecated_keys and not _DEPRECATION_NOTIFIED:
|
|
147
|
+
message = (
|
|
148
|
+
"Deprecated config keys "
|
|
149
|
+
+ ", ".join(sorted(deprecated_keys))
|
|
150
|
+
+ " found; prefer 'rest_api_transport_enabled' and 'residential_proxy_enabled'."
|
|
151
|
+
)
|
|
152
|
+
logger.warning(message)
|
|
153
|
+
warnings.warn(message, DeprecationWarning, stacklevel=2)
|
|
154
|
+
_DEPRECATION_NOTIFIED = True
|
|
155
|
+
if "rest_api_transport_enabled" in cfg.get("features", {}):
|
|
156
|
+
features["use_proxies"] = bool(features.get("rest_api_transport_enabled", False))
|
|
157
|
+
elif "use_proxies" in cfg.get("features", {}):
|
|
158
|
+
features["use_proxies"] = bool(cfg["features"].get("use_proxies", False))
|
|
159
|
+
|
|
160
|
+
if "residential_proxy_enabled" in cfg.get("features", {}):
|
|
161
|
+
features["use_proxy"] = bool(features.get("residential_proxy_enabled", False))
|
|
162
|
+
elif "use_proxy" in cfg.get("features", {}):
|
|
163
|
+
features["use_proxy"] = bool(cfg["features"].get("use_proxy", False))
|
|
130
164
|
if cfg.get("output_dir"):
|
|
131
165
|
merged["output_dir"] = cfg["output_dir"]
|
|
132
166
|
return merged
|
|
133
167
|
|
|
134
168
|
|
|
135
169
|
def save_config(config: dict[str, Any]) -> None:
|
|
170
|
+
features = config.setdefault("features", {})
|
|
171
|
+
features["rest_api_transport_enabled"] = bool(features.get("rest_api_transport_enabled", features.get("use_proxies", False)))
|
|
172
|
+
features["residential_proxy_enabled"] = bool(features.get("residential_proxy_enabled", features.get("use_proxy", False)))
|
|
173
|
+
features["use_proxies"] = features["rest_api_transport_enabled"]
|
|
174
|
+
features["use_proxy"] = features["residential_proxy_enabled"]
|
|
136
175
|
_ensure_home()
|
|
137
176
|
CONFIG_PATH.write_text(
|
|
138
177
|
json.dumps(config, indent=2, sort_keys=True),
|
|
@@ -291,9 +330,11 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
|
|
|
291
330
|
# gates (see sources/proxy_client.py §Architectural grounding for
|
|
292
331
|
# the full quote from https://docs.scrapingant.com/proxy-mode).
|
|
293
332
|
#
|
|
294
|
-
# - features.
|
|
333
|
+
# - features.rest_api_transport_enabled (preferred) →
|
|
334
|
+
# VOIDACCESS_USE_PROXIES=true
|
|
295
335
|
# Selects the REST API transport.
|
|
296
|
-
# - features.
|
|
336
|
+
# - features.residential_proxy_enabled (preferred) →
|
|
337
|
+
# VOIDACCESS_USE_PROXY=true
|
|
297
338
|
# Selects the proxy transport.
|
|
298
339
|
#
|
|
299
340
|
# Each is set ONLY when the user has explicitly enabled it. The
|
|
@@ -302,11 +343,11 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
|
|
|
302
343
|
# fallback surprise. This is the same logic already proven in
|
|
303
344
|
# Phase 1's tests.
|
|
304
345
|
features = cfg.get("features") or {}
|
|
305
|
-
if features.get("use_proxies"):
|
|
346
|
+
if features.get("rest_api_transport_enabled", features.get("use_proxies")):
|
|
306
347
|
os.environ["VOIDACCESS_USE_PROXIES"] = "true"
|
|
307
348
|
else:
|
|
308
349
|
os.environ.pop("VOIDACCESS_USE_PROXIES", None)
|
|
309
|
-
if features.get("use_proxy"):
|
|
350
|
+
if features.get("residential_proxy_enabled", features.get("use_proxy")):
|
|
310
351
|
os.environ["VOIDACCESS_USE_PROXY"] = "true"
|
|
311
352
|
else:
|
|
312
353
|
os.environ.pop("VOIDACCESS_USE_PROXY", None)
|
|
@@ -206,7 +206,7 @@ def status(
|
|
|
206
206
|
# transport selector so the display ALWAYS matches what
|
|
207
207
|
# clearnet_fetch() will actually do. Never reveals raw keys.
|
|
208
208
|
#
|
|
209
|
-
# both off → red "disabled" (direct —
|
|
209
|
+
# both off → red "disabled" (direct — pre-1.6.2 behavior)
|
|
210
210
|
# proxy only → cyan "proxy transport" (HTTP CONNECT through the configured ScrapingAnt proxy endpoint)
|
|
211
211
|
# api only → green "REST API" (POST api.scrapingant.com/v2/general)
|
|
212
212
|
# both on (proxy wins) → magenta "proxy transport (api would be overridden)"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0002_add_investigation_status_column.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0007_add_actor_style_profiles.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0011_add_page_extraction_cache.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0018_add_user_id_to_investigations.py
RENAMED
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0020_add_entity_source_tracking.py
RENAMED
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0021_add_search_engine_stats.py
RENAMED
|
File without changes
|
|
File without changes
|
{voidaccess-1.6.2 → voidaccess-1.6.3}/db/migrations/versions/0023_add_investigation_metadata.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|