torbot 4.0.0__tar.gz → 4.3.0__tar.gz

torbot-4.3.0/.env

@@ -0,0 +1 @@
+export TORBOT_DATA_DIR=${PWD}/data

torbot-4.3.0/.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 119
+exclude = .venv,build,dist

torbot-4.3.0/.gitattributes

@@ -0,0 +1,21 @@
+# Source files
+# ============
+*.pxd		text
+*.py 		text
+*.py3 		text
+*.pyw 		text
+*.pyx  		text
+
+# Binary files
+# ============
+*.db		binary
+*.p 		binary
+*.pkl 		binary
+*.pyc 		binary
+*.pyd		binary
+*.pyo 		binary
+
+# Note: .db, .p, and .pkl files are associated
+# with the python modules ``pickle``, ``dbm.*``,
+# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``
+# (among others).

torbot-4.3.0/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: psnappz
+patreon: dedsec_inside_research
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with a single custom sponsorship URL

torbot-4.3.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

torbot-4.3.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

torbot-4.3.0/.github/pull_request_template.md

@@ -0,0 +1,12 @@
+Issue #
+
+### Changes Proposed
+- 
+- 
+- 
+
+### Explanation of Changes
+
+
+
+### Screenshots of new feature/change

torbot-4.3.0/.github/workflows/flake8.yml

@@ -0,0 +1,26 @@
+name: Flake8
+
+on: [push]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8
+    - name: Analysing the code with flake8
+      run: |
+        flake8 --per-file-ignores="__init__.py:F401" .

torbot-4.3.0/.github/workflows/pytest.yml

@@ -0,0 +1,34 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Pytest
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -e .
+    - name: Test with pytest
+      run: |
+        pytest

torbot-4.3.0/.gitignore

@@ -0,0 +1,38 @@
+# Module Ignores
+modules/*.pyc
+modules/__init__.py
+modules/__pycache__/
+modules/.pytest_cache
+modules/.ropeproject/
+modules/lib/*.so
+
+# Tests Ignores
+tests/.pytest_cache
+tests/*.pyc
+tests/__pycache__/
+tests/.ropeproject/
+
+# Cache
+.cache
+*.pyc
+.pytestc*
+.pytest_cache
+__pycache*
+__pycache__/
+
+# Misc
+
+.*.swp
+.ropeproject/
+.idea/
+.vscode/
+.DS_Store
+
+venv/
+.venv/
+.DS_Store
+data/
+torbot/modules/nlp/training_data/
+
+# Deployment artifacts
+dist

torbot-4.3.0/CITATION.cff

@@ -0,0 +1,40 @@
+# @InProceedings{10.1007/978-981-15-0146-3_19,
+# author="Narayanan, P. S.
+# and Ani, R.
+# and King, Akeem T. L.",
+# editor="Ranganathan, G.
+# and Chen, Joy
+# and Rocha, {\'A}lvaro",
+# title="TorBot: Open Source Intelligence Tool for Dark Web",
+# booktitle="Inventive Communication and Computational Technologies",
+# year="2020",
+# publisher="Springer Singapore",
+# address="Singapore",
+# pages="187--195",
+# abstract="The dark web has turned into a dominant source of illegal activities. With several volunteered networks, it is      becoming more difficult to track down these services. Open source intelligence (OSINT) is a technique used to gather intelligence on targets by harvesting publicly available data. Performing OSINT on the Tor network makes it a challenge for both researchers and developers because of the complexity and anonymity of the network. This paper presents a tool which shows OSINT in the dark web. With the use of this tool, researchers and Law Enforcement Agencies can automate their task of crawling and identifying different services in the Tor network. This tool has several features which can help extract different intelligence.",
+# isbn="978-981-15-0146-3"
+# }
+
+cff-version: 1.2.0
+message: "If you use this software, please cite the following paper:"
+authors:
+  - family-names: P. S.
+    given-names: Narayanan
+    affiliation: Department of Computer Science and Applications, Amrita Vishwa Vidyapeetham, Amritapuri, Kollam, India
+  - family-names: Akeem T. L.
+    given-names: King
+    affiliation: USPA Technologies
+  - family-names: R
+    given-names: Ani
+    affiliation: Department of Computer Science and Applications, Amrita Vishwa Vidyapeetham, Amritapuri, Kollam, India
+keywords:
+  - tor
+  - research
+  - osint
+identifiers:
+  - type: doi
+    value: 10.1007/978-981-15-0146-3_19
+license: GNU Public License
+reposiory-code: https://github.com/DedSecInside/TorBot
+title: TorBot - Open Source Intelligence Tool for Dark Web
+date-released: 2020-01-30

torbot-4.3.0/Dockerfile

@@ -0,0 +1,22 @@
+# Use an official Python 3.11.4 image as the base
+FROM python:3.11.4
+
+# Set a working directory within the container
+WORKDIR /app
+
+# Clone the TorBot repository from GitHub
+RUN git clone https://github.com/DedSecInside/TorBot.git /app
+
+# Install dependencies
+RUN pip install -r /app/requirements.txt
+
+# Set the SOCKS5_PORT environment variable
+ENV SOCKS5_PORT=9050
+
+# Expose the port specified in the .env file
+EXPOSE $SOCKS5_PORT
+
+# Run the TorBot script
+CMD ["poetry", "run", "python", "torbot"]
+# Example way to run the container:
+# docker run --network="host" your-image-name poetry run python torbot -u https://www.example.com --depth 2 --visualize tree --save json

torbot-4.3.0/PKG-INFO

@@ -0,0 +1,258 @@
+Metadata-Version: 2.4
+Name: torbot
+Version: 4.3.0
+Summary: TorBot is an OSINT tool for the dark web.
+Author-email: Akeem King <akeemtlking@gmail.com>, PS Narayanan <thepsnarayanan@gmail.com>
+License-File: LICENSE.md
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: altgraph>=0.17.4
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: certifi>=2024.7.4
+Requires-Dist: charset-normalizer>=3.3.2
+Requires-Dist: decorator>=5.1.1
+Requires-Dist: httpx[socks]>=0.27.0
+Requires-Dist: idna>=3.15
+Requires-Dist: igraph>=0.11.4
+Requires-Dist: joblib>=1.4.2
+Requires-Dist: macholib>=1.16.3
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: phonenumbers>=8.13.37
+Requires-Dist: progress>=1.6
+Requires-Dist: pyinstaller-hooks-contrib>=2024.6
+Requires-Dist: pyinstaller>=6.8.0
+Requires-Dist: pysocks>=1.7.1
+Requires-Dist: pytest>=9.0.3
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: scikit-learn>=1.5.0
+Requires-Dist: scipy>=1.13.0
+Requires-Dist: six>=1.16.0
+Requires-Dist: soupsieve>=2.8.4
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: termcolor>=2.4.0
+Requires-Dist: texttable>=1.7.0
+Requires-Dist: threadpoolctl>=3.5.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: treelib>=1.7.1
+Requires-Dist: unipath>=1.1
+Requires-Dist: urllib3>=2.7.0
+Requires-Dist: validators>=0.22.0
+Requires-Dist: yattag>=1.15.2
+Description-Content-Type: text/markdown
+
+ <pre>
+
+                         ████████╗ ██████╗ ██████╗     ██████╗  ██████╗ ████████╗
+                         ╚══██╔══╝██╔═══██╗██╔══██╗    ██╔══██╗██╔═████╗╚══██╔══╝
+                            ██║   ██║   ██║██████╔╝    ██████╔╝██║██╔██║   ██║
+                            ██║   ██║   ██║██╔══██╗    ██╔══██╗████╔╝██║   ██║
+                            ██║   ╚██████╔╝██║  ██║    ██████╔╝╚██████╔╝   ██║
+                            ╚═╝    ╚═════╝ ╚═╝  ╚═╝    ╚═════╝  ╚═════╝    ╚═╝
+
+                            Open Source Intelligence Tool for the Dark Web
+
+</pre>
+
+<p align="center">
+   <a href="https://owasp.org/www-project-torbot/">
+    <img src="https://user-images.githubusercontent.com/4481429/94645022-65f35680-0308-11eb-8dce-a4ceba8db660.png" width="270" title="OWASP Foundation">
+   </a>
+   <img src="https://github.com/DedSecInside/TorBot/assets/4481429/41f5e7ea-56fe-41b0-bd26-2788deeb5f10" width="250" title="Hacktoberfest 2022">
+   <img src="https://user-images.githubusercontent.com/4481429/94646312-9dafcd80-030b-11eb-9824-73aa2414cf58.png" width="160" title="OpenSource">
+</p>
+
+
+### Status/Social links
+[![Pytest](https://github.com/DedSecInside/TorBot/actions/workflows/pytest.yml/badge.svg)](https://github.com/DedSecInside/TorBot/actions/workflows/pytest.yml)
+[![Flake8](https://github.com/DedSecInside/TorBot/actions/workflows/flake8.yml/badge.svg?branch=dev)](https://github.com/DedSecInside/TorBot/actions/workflows/flake8.yml)
+[![Slack Invite](https://img.shields.io/badge/Join-Slack-informational?style=flat-square&logo=slack)](https://join.slack.com/t/dedsec-inside/shared_invite/zt-i4jcgj8w-1ecHiSdRubwuppAzz8~yfg)
+[![Code Triage](https://www.codetriage.com/dedsecinside/torbot/badges/users.svg)](https://www.codetriage.com/dedsecinside/torbot)
+[![](https://img.shields.io/badge/Built%20with-❤-orange.svg?style=flat-square)]()
+[![](https://img.shields.io/badge/Made%20with-Python-red.svg?style=flat-square)]()
+
+### Features
+1. Onion Crawler (.onion)
+2. Returns page title or host name if no page title is available and address with a short description of the site.
+3. Save links to a database (Not done)
+4. Output the HTML from a site or save it to an HTML file. (Not done)
+5. Save the link tree as a JSON file.
+6. Crawl custom domains
+7. Check if the link is live
+8. Built-in Updater
+9. Build a visual tree of link relationships that can be quickly viewed or saved to a file
+
+### Recent improvements
+- The CLI now allows `--version` and `--update` to run without requiring a URL.
+- The crawler now resolves relative links against the current page and skips unsupported or malformed hrefs more gracefully.
+- The optional TorBotApp desktop UI can be launched from the CLI with `torbot app` when installed as a sibling checkout.
+
+...(will be updated)
+
+### Dependencies
+- Tor (Optional)
+- Python 3.9+
+- pip
+
+## Quick start
+
+### Local setup
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -r requirements.txt
+python -m pip install -e .
+```
+
+### Run the CLI
+```sh
+python main.py --help
+python main.py --version
+python main.py -u https://example.com --depth 2 --visualize table
+```
+
+### Common examples
+```sh
+# Gather basic site information
+python main.py -u https://example.com --info
+
+# Save the crawl tree as JSON
+python main.py -u https://example.com --depth 2 --save json
+
+# Disable SOCKS5 if you are not using Tor locally
+python main.py -u https://example.com --disable-socks5 --visualize tree
+
+# Launch the optional desktop app when TorBotApp is installed
+torbot app
+```
+
+### Options
+```text
+usage: Gather and analyze data from Tor sites.
+
+optional arguments:
+  -u URL, --url URL     Specify a website link to crawl
+  --depth DEPTH         Specify max depth of crawler (default 1)
+  -h, --help            Show this help message and exit
+  --host                Set IP address for SOCKS5 proxy (defaults to 127.0.0.1)
+  --port                Set port for SOCKS5 proxy (defaults to 9050)
+  -v                    Displays DEBUG level logging, default is INFO
+  --version             Show the current version of TorBot.
+  --update              Update TorBot to the latest stable version
+  --app                 Run optional TorBot desktop app
+  --app-dir APP_DIR     Path to a TorBotApp checkout
+  -q, --quiet           Prevents display of header and IP address
+  --save FORMAT         Save results in a file. (tree, JSON)
+  --visualize FORMAT    Visualizes tree of data gathered. (tree, JSON, table)
+  -i, --info            Info displays basic info of the scanned site
+  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy
+```
+
+> `-u/--url` is required for crawl-related commands such as `--info`, `--save`, and `--visualize`.
+
+If you are using Tor locally, keep the SOCKS5 proxy enabled. If you are not using Tor, add `--disable-socks5` to avoid proxy connection errors.
+
+Read more about torrc here: [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md)
+
+## Optional desktop app
+
+TorBot remains a CLI-first Python tool. The desktop UI lives in the separate
+[TorBotApp](https://github.com/KingAkeem/TorBotApp) Electron project so Node and
+Electron are not required for CLI users.
+
+To launch it from this CLI, install TorBotApp as a sibling checkout:
+
+```text
+code/
++-- TorBot/
++-- TorBotApp/
+```
+
+Then run:
+
+```sh
+torbot app
+```
+
+If TorBotApp is somewhere else, use either:
+
+```sh
+TORBOT_APP_DIR=/path/to/TorBotApp torbot app
+torbot --app --app-dir /path/to/TorBotApp
+```
+
+See [Desktop App Architecture](docs/DESKTOP_APP.md) for how the CLI, optional UI,
+and backend services fit together.
+
+## Curated Features
+- [x] Visualization Module Revamp
+- [x] Implement BFS Search for webcrawler
+- [x] Improve stability (Handle errors gracefully, expand test coverage, etc.)
+- [x] Increase test coverage
+- [ ] Save the most recent search results to a database
+- [ ] Randomize Tor Connection (Random Header and Identity)
+- [ ] Keyword/Phrase Search
+- [ ] Social Media Integration
+- [ ] Increase anonymity
+- [ ] Screenshot capture
+
+
+## Contribution Guidelines
+
+### Found an issue?
+
+If you face any issues in the project, please let us know by creating a new issue [here](https://github.com/DedSecInside/TorBot/issues/new/choose).
+
+### Developer Guidelines
+
+We welcome contributions to this project! Here are a few guidelines to follow:
+
+1. Fork the repository and create a new branch for your contribution.
+2. Make sure your code passes all tests by running `pytest` before submitting a pull request to `dev` branch.
+3. Follow the [PEP8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code.
+4. Make sure to add appropriate documentation for any new features or changes.
+5. When submitting a pull request, please provide a detailed description of the changes made.
+
+## References
+
+    1.  M. Glassman and M. J. Kang, “Intelligence in the internet age: The emergence and evolution of Open Source Intelligence (OSINT),” Comput. Human Behav., vol. 28, no. 2, pp. 673–682, 2012.
+    2.  D. Bradbury, “In plain view: open source intelligence,” Comput. Fraud Secur., vol. 2011, no. 4, pp. 5–9, 2011.
+    3.  B. Butler, B. Wardman, and N. Pratt, “REAPER: an automated, scalable solution for mass credential harvesting and OSINT,” 2016 APWG Symp. Electron. Crime Res., pp. 1–10, 2016.
+    4.  B. Zantout and R. A. Haraty, “I2P Data Communication System I2P Data Communication System,” no. April 2002, 2014.
+    5.  J. Qin, Y. Zhou, G. Lai, E. Reid, M. Sageman, and H. Chen, “The dark web portal project: collecting and analyzing the presence of terrorist groups on the web,” in Proceedings of the 2005 IEEE international conference on Intelligence and Security Informatics, 2005, pp. 623–624.
+    6.  D. Moore, T. Rid, D. Moore, and T. Rid, “Cryptopolitik and the Darknet Cryptopolitik and the Darknet,” vol. 6338, 2016.
+    7.  G. Weimann, “Going dark: Terrorism on the dark Web,” Stud. Confl. Terror., vol. 39, no. 3, pp. 195–206, 2016.
+    8.  A. T. Zulkarnine, R. Frank, B. Monk, J. Mitchell, and G. Davies, “Surfacing collaborated networks in dark web to find illicit and criminal content,” in Intelligence and Security Informatics (ISI), 2016 IEEE Conference on, 2016, pp. 109–114.
+    9.  T. Minárik and A.-M. Osula, “Tor does not stink: Use and abuse of the Tor anonymity network from the perspective of law,” Comput. Law Secur. Rev., vol. 32, no. 1, pp. 111–127, 2016.
+    10. K. Loesing, S. J. Murdoch, and R. Dingledine, “A Case Study on Measuring Statistical Data in the {T}or Anonymity Network,” in Proceedings of the Workshop on Ethics in Computer Security Research (WECSR 2010), 2010.
+    11. B. Nafziger, “Data Mining in the Dark : Darknet Intelligence Automation,” 2017.
+    12. I. Sanchez-Rola, D. Balzarotti, and I. Santos, “The onions have eyes: A comprehensive structure and privacy analysis of tor hidden services,” in Proceedings of the 26th International Conference on World Wide Web, 2017, pp. 1251–1260.
+    13. Mouli VR, Jevitha KP. “Web Services Attacks and Security-A Systematic Literature Review.”, Procedia Computer Science. 2016 Jan 1;93:870-7.
+    14. Cova M, Felmetsger V, Vigna G. "Vulnerability analysis of web-based applications. InTest and Analysis of Web Services" 2007 (pp. 363-394). Springer, Berlin, Heidelberg.
+    15. B. R. Holland, “Enabling Open Source Intelligence (OSINT) in private social networks,” 2012.
+    16. S. Nakamoto, “Bitcoin: A Peer-to-Peer Electronic Cash System,” Cryptogr. Mail. List https//metzdowd.com, 2009.
+    17. M. Wesam, A. Nabki, E. Fidalgo, E. Alegre, and I. De Paz, “Classifying Illegal Activities on Tor Network Based on Web Textual Contents”, vol. 1, pp. 35–43, 2017.
+    18. Sathyadevan S, Gangadharan S.“Crime analysis and prediction using data mining”. In Networks & Soft Computing (ICNSC), 2014 First International Conference on 2014 Aug 19 (pp. 406-412). IEEE.
+    19. Chau M, Chen H. "A machine learning approach to web page filtering using content and structure analysis. Decision Support Systems." 2008 Jan 1;44(2):482-94.
+    20. Ani R, Jose J, Wilson M, Deepa OS. “Modified Rotation Forest Ensemble Classifier for Medical Diagnosis in Decision Support Systems”, In Progress in Advanced Computing and Intelligent Engineering 2018 (pp. 137-146). Springer, Singapore.
+    21. Ani R, Augustine A, Akhil N.C. and Deepa O.S., 2016. “Random Forest Ensemble Classifier to Predict the Coronary Heart Disease Using Risk Factors”, In Proceedings of the International Conference on Soft Computing Systems (pp. 701-710). Springer, New Delhi.
+
+
+## Maintainers
+
+- [X] [PS Narayanan](https://github.com/PSNAppz) - Co-owner
+- [X] [KingAkeem](https://github.com/KingAkeem) - Co-owner
+
+## All Thanks to Our Contributors
+
+<a href="https://github.com/DedSecInside/TorBot/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=DedSecInside/TorBot" />
+</a>
+
+## License
+[GNU Public License](https://github.com/DedSecInside/TorBot/blob/dev/LICENSE.md)