torbot 4.3.0__tar.gz → 4.3.1__tar.gz

torbot-4.3.1/.github/workflows/flake8.yml

@@ -0,0 +1,33 @@
+name: Flake8
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8
+    - name: Analysing the code with flake8
+      run: flake8 --per-file-ignores="__init__.py:F401" .

torbot-4.3.1/.github/workflows/pytest.yml

@@ -0,0 +1,37 @@
+name: Pytest
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: |
+          requirements.txt
+          pyproject.toml
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r requirements.txt
+        python -m pip install --no-deps -e .
+    - name: Test with pytest
+      run: pytest

{torbot-4.3.0 → torbot-4.3.1}/PKG-INFO

@@ -1,16 +1,15 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: torbot
-Version: 4.3.0
+Version: 4.3.1
 Summary: TorBot is an OSINT tool for the dark web.
 Author-email: Akeem King <akeemtlking@gmail.com>, PS Narayanan <thepsnarayanan@gmail.com>
 License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: altgraph>=0.17.4
 Requires-Dist: beautifulsoup4>=4.12.3
 Requires-Dist: certifi>=2024.7.4
@@ -95,7 +94,7 @@ Description-Content-Type: text/markdown
 
 ### Dependencies
 - Tor (Optional)
-- Python 3.9+
+- Python 3.10+
 - pip
 
 ## Quick start
@@ -109,6 +108,16 @@ python -m pip install -r requirements.txt
 python -m pip install -e .
 ```
 
+### Install with OTinstaller (Linux)
+[OTinstaller](https://github.com/OSINTVerseHQ/OTinstaller) is a third-party tool manager that installs the `torbot` PyPI release into its own virtualenv. It is not maintained by the TorBot team, so please report problems with it [on the OTinstaller tracker](https://github.com/OSINTVerseHQ/OTinstaller/issues).
+```sh
+pipx install otinstaller
+otinstaller init
+otinstaller install torbot
+# url:... is passed to TorBot as -u; the other flags go through unchanged
+otinstaller run torbot -- url:https://example.com --depth 2 --visualize table
+```
+
 ### Run the CLI
 ```sh
 python main.py --help
@@ -131,6 +140,24 @@ python main.py -u https://example.com --disable-socks5 --visualize tree
 torbot app
 ```
 
+### Evidence-first AI analysis
+
+Save a versioned crawl result and turn it into a cited investigation bundle:
+
+```sh
+torbot --url https://example.com --disable-socks5 \
+  --save result --result-file crawl-result.json
+torbot analyze crawl-result.json --provider ollama --model qwen3 \
+  --output investigation/
+```
+
+Analyst defaults to a local Ollama endpoint. Remote OpenAI-compatible
+providers require the explicit `--allow-remote` flag. Every supported finding
+must cite captured evidence, and a provider outage never prevents the
+deterministic evidence bundle from being written. See
+[TorBot Analyst](docs/ANALYST.md) for the offline quick start, output formats,
+privacy boundary, and schemas.
+
 ### Options
 ```text
 usage: Gather and analyze data from Tor sites.

{torbot-4.3.0 → torbot-4.3.1}/README.md

@@ -48,7 +48,7 @@
 
 ### Dependencies
 - Tor (Optional)
-- Python 3.9+
+- Python 3.10+
 - pip
 
 ## Quick start
@@ -62,6 +62,16 @@ python -m pip install -r requirements.txt
 python -m pip install -e .
 ```
 
+### Install with OTinstaller (Linux)
+[OTinstaller](https://github.com/OSINTVerseHQ/OTinstaller) is a third-party tool manager that installs the `torbot` PyPI release into its own virtualenv. It is not maintained by the TorBot team, so please report problems with it [on the OTinstaller tracker](https://github.com/OSINTVerseHQ/OTinstaller/issues).
+```sh
+pipx install otinstaller
+otinstaller init
+otinstaller install torbot
+# url:... is passed to TorBot as -u; the other flags go through unchanged
+otinstaller run torbot -- url:https://example.com --depth 2 --visualize table
+```
+
 ### Run the CLI
 ```sh
 python main.py --help
@@ -84,6 +94,24 @@ python main.py -u https://example.com --disable-socks5 --visualize tree
 torbot app
 ```
 
+### Evidence-first AI analysis
+
+Save a versioned crawl result and turn it into a cited investigation bundle:
+
+```sh
+torbot --url https://example.com --disable-socks5 \
+  --save result --result-file crawl-result.json
+torbot analyze crawl-result.json --provider ollama --model qwen3 \
+  --output investigation/
+```
+
+Analyst defaults to a local Ollama endpoint. Remote OpenAI-compatible
+providers require the explicit `--allow-remote` flag. Every supported finding
+must cite captured evidence, and a provider outage never prevents the
+deterministic evidence bundle from being written. See
+[TorBot Analyst](docs/ANALYST.md) for the offline quick start, output formats,
+privacy boundary, and schemas.
+
 ### Options
 ```text
 usage: Gather and analyze data from Tor sites.

torbot-4.3.1/docs/ANALYST.md

@@ -0,0 +1,82 @@
+# TorBot Analyst
+
+TorBot Analyst turns a versioned crawl result into a local investigation
+bundle. It records stable evidence IDs and content hashes, maps page and
+contact relationships, and requires every supported finding to reference
+captured evidence.
+
+## Five-minute quick start
+
+Create a deterministic report without an AI provider:
+
+```sh
+torbot analyze tests/fixtures/analyst-safe-crawl.json \
+  --provider none \
+  --keyword example \
+  --output investigation/
+```
+
+The output contains:
+
+- `report.md` with supported, conflicted, and unsupported findings
+- `evidence.jsonl` with captured URLs, excerpts, timestamps, and hashes
+- `graph.json` with evidence-linked page and contact relationships
+- `run.json` with reproducibility metadata and provider warnings
+
+## Capture a crawl result
+
+The existing tree and JSON outputs are unchanged. Save the new versioned
+result explicitly:
+
+```sh
+torbot --url https://example.com \
+  --disable-socks5 \
+  --save result \
+  --result-file crawl-result.json
+```
+
+The versioned result contains bounded visible text, not raw HTML, scripts,
+styles, cookies, headers, credentials, or arbitrary exception messages.
+
+## Local AI analysis
+
+Ollama is the default provider and is contacted only on localhost:
+
+```sh
+torbot analyze crawl-result.json \
+  --provider ollama \
+  --model qwen3 \
+  --output investigation/
+```
+
+If Ollama or the model is unavailable, TorBot still writes the deterministic
+evidence bundle and records a warning in `run.json` and `report.md`.
+
+## Remote OpenAI-compatible providers
+
+Remote transmission is fail-closed. Both an explicit provider and
+`--allow-remote` are required. The API key is read from `OPENAI_API_KEY` and is
+never written to an output file. Email addresses and phone numbers are
+redacted before remote transmission. Repeat `--redact-pattern` to add
+case-specific regular expressions.
+
+```sh
+export OPENAI_API_KEY=your-key
+torbot analyze crawl-result.json \
+  --provider openai-compatible \
+  --base-url https://api.openai.com/v1 \
+  --model your-model \
+  --allow-remote \
+  --redact-pattern 'CASE-[0-9]+' \
+  --output investigation/
+```
+
+Captured page text is untrusted data. Analyst exposes no tools to the model,
+does not execute page instructions, discards unknown evidence references, and
+never promotes an uncited model statement to `supported`.
+
+## Schemas
+
+Machine-readable schemas live in `schemas/crawl-result.v1.schema.json` and
+`schemas/investigation-report.v1.schema.json`. Consumers must reject schema
+versions they do not understand.

{torbot-4.3.0 → torbot-4.3.1}/docs/CHANGELOG.md

@@ -2,6 +2,25 @@
 --------------------
 All notable changes to this project will be documented in this file.
 
+## 4.3.1 - 2026-11-12
+
+### Added
+- Added `--save result` for analysis-ready crawl-result v1 files with bounded visible text and content hashes.
+- Added `torbot analyze` for deterministic evidence bundles and optional local or remote OpenAI-compatible analysis.
+- Added stable evidence IDs, claim citation validation, an evidence graph, JSON schemas, and a safe surface-web fixture.
+
+### Changed
+- Requires Python 3.10 or newer, with CI coverage for Python 3.10, 3.11, and 3.12.
+- Updated pinned AnyIO, Soup Sieve, urllib3, certifi, and setuptools dependencies.
+- Documented installation with OTinstaller.
+- Excluded environment files from package builds.
+
+### Security
+- Fixed path traversal in tree, JSON, and HTML export filenames by sanitizing untrusted page titles. Thanks to Artur Chlebicki (@Artur12555) for reporting GHSA-5888-59xj-84pf.
+- Added regression tests for relative and absolute paths, Windows separators, encoded separators, reserved names, and long titles.
+- Remote evidence transmission now requires `--allow-remote`, redacts contact data by default, and never stores API keys.
+- Crawled content is treated as untrusted data, model calls receive no tools, and uncited model output cannot become a supported finding.
+
 ## 4.3.0 - 2026-07-28
 
 ### Changed

torbot-4.3.1/docs/CRAWL_RESULT.md

@@ -0,0 +1,41 @@
+# Shared crawl-result contract
+
+TorBot exposes a versioned, machine-readable crawl result through
+`LinkTree.to_crawl_result()`. It is an integration contract, not a replacement
+for the existing tree JSON saved by `--save json`; that legacy format remains
+available through the 4.x series. Consumers must feature-detect
+`schemaVersion` and reject versions they do not understand.
+
+## Version 1
+
+The top-level object contains `schemaVersion`, a unique `runId`, `engine` and
+`engineVersion`, normalized `target`, effective `settings`, terminal status,
+timestamps, `durationMs`, `pages`, and privacy-safe `diagnostics`.
+
+Each page records `url`, `parentUrl`, `depth`, `outcome`, HTTP `status`,
+`skippedReason`, bounded `errorCategory`, extracted `links`, and `contacts`.
+`outcome` is `fetched` or `failed`; failures expose one of `cancelled`,
+`invalid_input`, `network`, `parse`, or `unknown`, never an exception message.
+Links and contacts include their source (`anchor`, `mailto`, or `tel`). Raw
+HTML, headers, cookies, credentials, and arbitrary exception strings are not
+part of the contract.
+
+When a caller explicitly requests analysis-ready output, a fetched page may
+also include a `title`, deterministic classification metadata, and `content`.
+Content contains bounded visible plain text and its SHA-256 digest. Scripts,
+styles, templates, and raw markup are excluded. The CLI exposes this form only
+through `--save result`; legacy `--save json` behavior is unchanged.
+
+GoTor's versioned report is the compatibility baseline: both projects use
+`schemaVersion: 1`, `engine`, `target`, depth/settings, timestamps, duration,
+and per-page URL/parent/depth/status/link information. TorBot's fields are
+additive where GoTor's current report has no equivalent (`runId`, terminal
+state, provenance, bounded errors, and diagnostics). Consumers should retain
+unknown fields and branch only on the schema version, not on engine-specific
+internals.
+
+## Deprecation path
+
+No existing CLI output changes in this issue. A future major release may mark
+the legacy tree JSON as deprecated only after the shared result is available
+from an explicit CLI/API surface and downstream consumers have migrated.

{torbot-4.3.0 → torbot-4.3.1}/pyproject.toml

@@ -4,17 +4,16 @@ build-backend = "hatchling.build"
 
 [project]
 name = "torbot"
-version = "4.3.0"
+version = "4.3.1"
 authors = [
     { name="Akeem King", email="akeemtlking@gmail.com" },
     { name="PS Narayanan", email="thepsnarayanan@gmail.com" },
 ]
 description = "TorBot is an OSINT tool for the dark web."
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 classifiers = [
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -57,3 +56,6 @@ dependencies = [
 
 [project.scripts]
 torbot = "torbot.cli:main"
+
+[tool.hatch.build]
+exclude = [".env", ".env.*"]

{torbot-4.3.0 → torbot-4.3.1}/requirements.txt

@@ -1,7 +1,7 @@
 altgraph==0.17.4
-anyio==4.3.0
+anyio==4.14.2
 beautifulsoup4==4.12.3
-certifi==2024.7.4
+certifi==2026.10.19
 charset-normalizer==3.3.2
 colorama==0.4.6 ; sys_platform == "win32"
 decorator==5.1.1
@@ -27,11 +27,11 @@ python-dotenv==1.2.2
 pywin32-ctypes==0.2.2 ; sys_platform == "win32"
 scikit-learn==1.5.0
 scipy==1.13.0
-setuptools==83.0.0
+setuptools==84.1.0
 six==1.16.0
 sniffio==1.3.1
 socksio==1.0.0
-soupsieve==2.8.4
+soupsieve==2.9
 tabulate==0.9.0
 termcolor==2.4.0
 texttable==1.7.0
@@ -40,6 +40,6 @@ toml==0.10.2
 tomli==2.0.1
 treelib==1.7.1
 unipath==1.1
-urllib3==2.7.0
+urllib3==2.8.0
 validators==0.22.0
 yattag==1.15.2

torbot-4.3.1/schemas/crawl-result.v1.schema.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/DedSecInside/TorBot/schemas/crawl-result.v1.schema.json",
+  "title": "TorBot crawl-result v1",
+  "type": "object",
+  "required": ["schemaVersion", "target", "pages"],
+  "properties": {
+    "schemaVersion": {"const": 1},
+    "runId": {"type": "string"},
+    "target": {"type": "string", "minLength": 1},
+    "pages": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["url", "outcome"],
+        "properties": {
+          "url": {"type": "string", "minLength": 1},
+          "outcome": {"enum": ["fetched", "failed", "skipped"]},
+          "content": {
+            "type": "object",
+            "required": ["mediaType", "text", "sha256"],
+            "properties": {
+              "mediaType": {"const": "text/plain"},
+              "text": {"type": "string"},
+              "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}
+            },
+            "additionalProperties": false
+          }
+        }
+      }
+    }
+  }
+}

torbot-4.3.1/schemas/investigation-report.v1.schema.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/DedSecInside/TorBot/schemas/investigation-report.v1.schema.json",
+  "title": "TorBot investigation-report v1",
+  "type": "object",
+  "required": ["schema", "createdAt", "source", "claims", "evidenceIds", "warnings"],
+  "properties": {
+    "schema": {"const": "investigation-report.v1"},
+    "claims": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["text", "status", "evidenceIds", "source"],
+        "properties": {
+          "text": {"type": "string"},
+          "status": {"enum": ["supported", "unsupported", "conflicted"]},
+          "evidenceIds": {"type": "array", "items": {"type": "string"}},
+          "source": {"enum": ["deterministic", "model"]}
+        },
+        "additionalProperties": false
+      }
+    },
+    "evidenceIds": {"type": "array", "items": {"type": "string"}},
+    "warnings": {"type": "array", "items": {"type": "string"}}
+  }
+}