undergraves 0.1.0__tar.gz

undergraves-0.1.0/LICENSE

@@ -0,0 +1,19 @@
+ Copyright (c) 2026 Gabriel Augusto da Costa Dill
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.

undergraves-0.1.0/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: undergraves
+Version: 0.1.0
+Summary: Command-line web crawler for domain mapping and endpoint discovery
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Dynamic: license-file
+
+# Undergraves
+
+Command-line web crawler for domain mapping and endpoint discovery. Performs breadth-first search (BFS) restricted strictly to the target's base domain, collecting active URLs and page titles in real time.
+
+---
+
+## Installation
+
+### Via PyPI
+```bash
+pip install undergraves
+
+```
+
+### Local Installation (Development)
+
+```bash
+git clone https://github.com/GabrielDillDev/undergraves.git
+cd undergraves
+pip install -e .
+
+```
+
+---
+
+## Usage
+
+```bash
+undergraves <URL> [LEVEL]
+
+```
+
+If no scan level is provided, **T3** is used by default.
+
+### Scan Levels
+
+| Level | Page Limit |
+| --- | --- |
+| **T1** | 50 |
+| **T2** | 200 |
+| **T3** | 500 *(Default)* |
+| **T4** | 2000 |
+| **T5** | Unlimited |
+
+### Examples
+
+Default scan (T3 - 500 pages):
+
+```bash
+undergraves https://example.com
+
+```
+
+Quick scan (T1 - 50 pages):
+
+```bash
+undergraves https://example.com T1
+
+```
+
+Full scan (T5 - Runs until queue is empty):
+
+```bash
+undergraves https://example.com T5
+
+```
+
+---
+
+## Output Files
+
+Upon completion, the tool automatically generates two files in the current working directory:
+
+* `resultado.json`: Array of objects containing `url` and `title`.
+* `resultado.csv`: Table formatted with `URL` and `Title` columns.
+
+---
+
+## Testing
+
+Run the unit test suite using `pytest`:
+
+```bash
+pip install pytest
+pytest
+
+```
+
+---
+
+## License
+
+MIT

undergraves-0.1.0/README.md

@@ -0,0 +1,93 @@
+# Undergraves
+
+Command-line web crawler for domain mapping and endpoint discovery. Performs breadth-first search (BFS) restricted strictly to the target's base domain, collecting active URLs and page titles in real time.
+
+---
+
+## Installation
+
+### Via PyPI
+```bash
+pip install undergraves
+
+```
+
+### Local Installation (Development)
+
+```bash
+git clone https://github.com/GabrielDillDev/undergraves.git
+cd undergraves
+pip install -e .
+
+```
+
+---
+
+## Usage
+
+```bash
+undergraves <URL> [LEVEL]
+
+```
+
+If no scan level is provided, **T3** is used by default.
+
+### Scan Levels
+
+| Level | Page Limit |
+| --- | --- |
+| **T1** | 50 |
+| **T2** | 200 |
+| **T3** | 500 *(Default)* |
+| **T4** | 2000 |
+| **T5** | Unlimited |
+
+### Examples
+
+Default scan (T3 - 500 pages):
+
+```bash
+undergraves https://example.com
+
+```
+
+Quick scan (T1 - 50 pages):
+
+```bash
+undergraves https://example.com T1
+
+```
+
+Full scan (T5 - Runs until queue is empty):
+
+```bash
+undergraves https://example.com T5
+
+```
+
+---
+
+## Output Files
+
+Upon completion, the tool automatically generates two files in the current working directory:
+
+* `resultado.json`: Array of objects containing `url` and `title`.
+* `resultado.csv`: Table formatted with `URL` and `Title` columns.
+
+---
+
+## Testing
+
+Run the unit test suite using `pytest`:
+
+```bash
+pip install pytest
+pytest
+
+```
+
+---
+
+## License
+
+MIT

undergraves-0.1.0/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "undergraves"
+version = "0.1.0"
+description = "Command-line web crawler for domain mapping and endpoint discovery"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "requests>=2.31.0",
+    "beautifulsoup4>=4.12.0"
+]
+
+[project.scripts]
+undergraves = "undergraves.cli:run"

undergraves-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

undergraves-0.1.0/src/undergraves/cli.py

@@ -0,0 +1,62 @@
+import sys
+from urllib.parse import urlparse
+from undergraves.crawler import crawl
+from undergraves.exporter import save_to_json, save_to_csv
+
+VALID_LEVELS = ["T1", "T2", "T3", "T4", "T5"]
+
+def is_valid_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return bool(parsed.netloc and parsed.scheme in ["http", "https"])
+    except Exception:
+        return False
+
+def run():
+    if len(sys.argv) < 2:
+        print("Usage: undergraves <URL> [LEVEL]")
+        print("Example: undergraves https://example.com T2")
+        sys.exit(1)
+
+    target_url = sys.argv[1]
+
+    if not target_url.startswith("http://") and not target_url.startswith("https://"):
+        target_url = "https://" + target_url
+
+    if not is_valid_url(target_url):
+        print(f"[-] Invalid URL: {target_url}")
+        sys.exit(1)
+
+    user_level = "T3"
+
+    if len(sys.argv) > 2:
+        provided_level = sys.argv[2].upper()
+        if provided_level not in VALID_LEVELS:
+            print(f"[-] Invalid scan level: {sys.argv[2]}. Allowed: {', '.join(VALID_LEVELS)}")
+            sys.exit(1)
+        user_level = provided_level
+
+    print(f"[*] Starting Undergraves scan against {target_url} (Level {user_level})")
+
+    try:
+        pages = crawl(target_url, level=user_level)
+
+        if not pages:
+            print("[-] No targets mapped or host unreachable.")
+            sys.exit(0)
+
+        save_to_json(pages, "resultado.json")
+        save_to_csv(pages, "resultado.csv")
+
+        print(f"[+] Scan completed. {len(pages)} endpoints mapped.")
+        print("[+] Results saved to 'resultado.json' and 'resultado.csv'.")
+
+    except KeyboardInterrupt:
+        print("\n[-] Scan aborted by user.")
+        sys.exit(0)
+    except Exception as e:
+        print(f"\n[-] Fatal error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    run()

undergraves-0.1.0/src/undergraves/crawler.py

@@ -0,0 +1,69 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+SCAN_LEVELS = {
+    "T1": 50,
+    "T2": 200,
+    "T3": 500,
+    "T4": 2000,
+    "T5": None
+}
+
+def crawl(url: str, level: str = "T3") -> list:
+    max_pages = SCAN_LEVELS.get(level.upper(), 500)
+
+    visited = set()
+    to_visit = [url]
+    results = []
+    
+    domain_parse = urlparse(url).netloc
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+    }
+
+    while to_visit:
+        if max_pages is not None and len(visited) >= max_pages:
+            break
+
+        current_url = to_visit.pop(0)
+
+        if current_url in visited:
+            continue
+
+        try:
+            response = requests.get(current_url, headers=headers, timeout=5)
+            
+            if response.status_code != 200:
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+            
+        except Exception:
+            continue
+
+        visited.add(current_url)
+
+        title_text = soup.title.string.strip() if soup.title and soup.title.string else "N/A"
+
+        print(f"[200] {current_url} | Title: {title_text}")
+
+        results.append({
+            "url": current_url,
+            "title": title_text
+        })
+
+        for link in soup.find_all("a"):
+            href = link.get("href")
+
+            if not href:
+                continue
+
+            href = urljoin(current_url, href)
+            href_parse = urlparse(href).netloc
+
+            if href_parse == domain_parse and href not in visited and href not in to_visit:
+                to_visit.append(href)
+
+    return results

undergraves-0.1.0/src/undergraves/exporter.py

@@ -0,0 +1,17 @@
+import json
+import csv
+
+def save_to_json(data: list, filename: str = "resultado.json"):
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+
+def save_to_csv(data: list, filename: str = "resultado.csv"):
+    if not data:
+        return
+
+    headers = data[0].keys()
+
+    with open(filename, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=headers)
+        writer.writeheader()
+        writer.writerows(data)

undergraves-0.1.0/src/undergraves.egg-info/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: undergraves
+Version: 0.1.0
+Summary: Command-line web crawler for domain mapping and endpoint discovery
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Dynamic: license-file
+
+# Undergraves
+
+Command-line web crawler for domain mapping and endpoint discovery. Performs breadth-first search (BFS) restricted strictly to the target's base domain, collecting active URLs and page titles in real time.
+
+---
+
+## Installation
+
+### Via PyPI
+```bash
+pip install undergraves
+
+```
+
+### Local Installation (Development)
+
+```bash
+git clone https://github.com/GabrielDillDev/undergraves.git
+cd undergraves
+pip install -e .
+
+```
+
+---
+
+## Usage
+
+```bash
+undergraves <URL> [LEVEL]
+
+```
+
+If no scan level is provided, **T3** is used by default.
+
+### Scan Levels
+
+| Level | Page Limit |
+| --- | --- |
+| **T1** | 50 |
+| **T2** | 200 |
+| **T3** | 500 *(Default)* |
+| **T4** | 2000 |
+| **T5** | Unlimited |
+
+### Examples
+
+Default scan (T3 - 500 pages):
+
+```bash
+undergraves https://example.com
+
+```
+
+Quick scan (T1 - 50 pages):
+
+```bash
+undergraves https://example.com T1
+
+```
+
+Full scan (T5 - Runs until queue is empty):
+
+```bash
+undergraves https://example.com T5
+
+```
+
+---
+
+## Output Files
+
+Upon completion, the tool automatically generates two files in the current working directory:
+
+* `resultado.json`: Array of objects containing `url` and `title`.
+* `resultado.csv`: Table formatted with `URL` and `Title` columns.
+
+---
+
+## Testing
+
+Run the unit test suite using `pytest`:
+
+```bash
+pip install pytest
+pytest
+
+```
+
+---
+
+## License
+
+MIT

undergraves-0.1.0/src/undergraves.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+src/undergraves/__init__.py
+src/undergraves/cli.py
+src/undergraves/crawler.py
+src/undergraves/exporter.py
+src/undergraves.egg-info/PKG-INFO
+src/undergraves.egg-info/SOURCES.txt
+src/undergraves.egg-info/dependency_links.txt
+src/undergraves.egg-info/entry_points.txt
+src/undergraves.egg-info/requires.txt
+src/undergraves.egg-info/top_level.txt
+tests/test_crawler.py

undergraves-0.1.0/src/undergraves.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

undergraves-0.1.0/src/undergraves.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+undergraves = undergraves.cli:run

undergraves-0.1.0/src/undergraves.egg-info/requires.txt

@@ -0,0 +1,2 @@
+requests>=2.31.0
+beautifulsoup4>=4.12.0

undergraves-0.1.0/src/undergraves.egg-info/top_level.txt

@@ -0,0 +1 @@
+undergraves

undergraves-0.1.0/tests/test_crawler.py

@@ -0,0 +1,43 @@
+import os
+import json
+import csv
+from undergraves.crawler import SCAN_LEVELS
+from undergraves.cli import is_valid_url
+from undergraves.exporter import save_to_json, save_to_csv
+
+def test_scan_levels_configuration():
+    assert SCAN_LEVELS["T1"] == 50
+    assert SCAN_LEVELS["T2"] == 200
+    assert SCAN_LEVELS["T3"] == 500
+    assert SCAN_LEVELS["T4"] == 2000
+    assert SCAN_LEVELS["T5"] is None
+
+def test_url_validation():
+    assert is_valid_url("https://example.com") is True
+    assert is_valid_url("http://sub.domain.org/path") is True
+    assert is_valid_url("not_a_url") is False
+    assert is_valid_url("ftp://invalid-scheme.com") is False
+
+def test_exporter_json(tmp_path):
+    sample_data = [{"url": "https://example.com", "title": "Example"}]
+    file_path = tmp_path / "test_output.json"
+    
+    save_to_json(sample_data, str(file_path))
+    
+    assert os.path.exists(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+        assert data == sample_data
+
+def test_exporter_csv(tmp_path):
+    sample_data = [{"url": "https://example.com", "title": "Example"}]
+    file_path = tmp_path / "test_output.csv"
+    
+    save_to_csv(sample_data, str(file_path))
+    
+    assert os.path.exists(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        reader = list(csv.DictReader(f))
+        assert len(reader) == 1
+        assert reader[0]["url"] == "https://example.com"
+        assert reader[0]["title"] == "Example"