undergraves 0.1.0__py3-none-any.whl

undergraves/cli.py

@@ -0,0 +1,62 @@
+import sys
+from urllib.parse import urlparse
+from undergraves.crawler import crawl
+from undergraves.exporter import save_to_json, save_to_csv
+
+VALID_LEVELS = ["T1", "T2", "T3", "T4", "T5"]
+
+def is_valid_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return bool(parsed.netloc and parsed.scheme in ["http", "https"])
+    except Exception:
+        return False
+
+def run():
+    if len(sys.argv) < 2:
+        print("Usage: undergraves <URL> [LEVEL]")
+        print("Example: undergraves https://example.com T2")
+        sys.exit(1)
+
+    target_url = sys.argv[1]
+
+    if not target_url.startswith("http://") and not target_url.startswith("https://"):
+        target_url = "https://" + target_url
+
+    if not is_valid_url(target_url):
+        print(f"[-] Invalid URL: {target_url}")
+        sys.exit(1)
+
+    user_level = "T3"
+
+    if len(sys.argv) > 2:
+        provided_level = sys.argv[2].upper()
+        if provided_level not in VALID_LEVELS:
+            print(f"[-] Invalid scan level: {sys.argv[2]}. Allowed: {', '.join(VALID_LEVELS)}")
+            sys.exit(1)
+        user_level = provided_level
+
+    print(f"[*] Starting Undergraves scan against {target_url} (Level {user_level})")
+
+    try:
+        pages = crawl(target_url, level=user_level)
+
+        if not pages:
+            print("[-] No targets mapped or host unreachable.")
+            sys.exit(0)
+
+        save_to_json(pages, "resultado.json")
+        save_to_csv(pages, "resultado.csv")
+
+        print(f"[+] Scan completed. {len(pages)} endpoints mapped.")
+        print("[+] Results saved to 'resultado.json' and 'resultado.csv'.")
+
+    except KeyboardInterrupt:
+        print("\n[-] Scan aborted by user.")
+        sys.exit(0)
+    except Exception as e:
+        print(f"\n[-] Fatal error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    run()

undergraves/crawler.py

@@ -0,0 +1,69 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+SCAN_LEVELS = {
+    "T1": 50,
+    "T2": 200,
+    "T3": 500,
+    "T4": 2000,
+    "T5": None
+}
+
+def crawl(url: str, level: str = "T3") -> list:
+    max_pages = SCAN_LEVELS.get(level.upper(), 500)
+
+    visited = set()
+    to_visit = [url]
+    results = []
+    
+    domain_parse = urlparse(url).netloc
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+    }
+
+    while to_visit:
+        if max_pages is not None and len(visited) >= max_pages:
+            break
+
+        current_url = to_visit.pop(0)
+
+        if current_url in visited:
+            continue
+
+        try:
+            response = requests.get(current_url, headers=headers, timeout=5)
+            
+            if response.status_code != 200:
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+            
+        except Exception:
+            continue
+
+        visited.add(current_url)
+
+        title_text = soup.title.string.strip() if soup.title and soup.title.string else "N/A"
+
+        print(f"[200] {current_url} | Title: {title_text}")
+
+        results.append({
+            "url": current_url,
+            "title": title_text
+        })
+
+        for link in soup.find_all("a"):
+            href = link.get("href")
+
+            if not href:
+                continue
+
+            href = urljoin(current_url, href)
+            href_parse = urlparse(href).netloc
+
+            if href_parse == domain_parse and href not in visited and href not in to_visit:
+                to_visit.append(href)
+
+    return results

undergraves/exporter.py

@@ -0,0 +1,17 @@
+import json
+import csv
+
+def save_to_json(data: list, filename: str = "resultado.json"):
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+
+def save_to_csv(data: list, filename: str = "resultado.csv"):
+    if not data:
+        return
+
+    headers = data[0].keys()
+
+    with open(filename, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=headers)
+        writer.writeheader()
+        writer.writerows(data)

undergraves-0.1.0.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: undergraves
+Version: 0.1.0
+Summary: Command-line web crawler for domain mapping and endpoint discovery
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Dynamic: license-file
+
+# Undergraves
+
+Command-line web crawler for domain mapping and endpoint discovery. Performs breadth-first search (BFS) restricted strictly to the target's base domain, collecting active URLs and page titles in real time.
+
+---
+
+## Installation
+
+### Via PyPI
+```bash
+pip install undergraves
+
+```
+
+### Local Installation (Development)
+
+```bash
+git clone https://github.com/GabrielDillDev/undergraves.git
+cd undergraves
+pip install -e .
+
+```
+
+---
+
+## Usage
+
+```bash
+undergraves <URL> [LEVEL]
+
+```
+
+If no scan level is provided, **T3** is used by default.
+
+### Scan Levels
+
+| Level | Page Limit |
+| --- | --- |
+| **T1** | 50 |
+| **T2** | 200 |
+| **T3** | 500 *(Default)* |
+| **T4** | 2000 |
+| **T5** | Unlimited |
+
+### Examples
+
+Default scan (T3 - 500 pages):
+
+```bash
+undergraves https://example.com
+
+```
+
+Quick scan (T1 - 50 pages):
+
+```bash
+undergraves https://example.com T1
+
+```
+
+Full scan (T5 - Runs until queue is empty):
+
+```bash
+undergraves https://example.com T5
+
+```
+
+---
+
+## Output Files
+
+Upon completion, the tool automatically generates two files in the current working directory:
+
+* `resultado.json`: Array of objects containing `url` and `title`.
+* `resultado.csv`: Table formatted with `URL` and `Title` columns.
+
+---
+
+## Testing
+
+Run the unit test suite using `pytest`:
+
+```bash
+pip install pytest
+pytest
+
+```
+
+---
+
+## License
+
+MIT

undergraves-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+undergraves/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+undergraves/cli.py,sha256=zUZ0LNL_fd0mHqE-z8a7q7QH-9-VV-WjiUy_SpMHHfk,1898
+undergraves/crawler.py,sha256=neBGwHah--3_uuJlLIYbbK07HlGEamHxzPR4BqEtfvo,1723
+undergraves/exporter.py,sha256=uvVNKP92GsiX_aILnF-SynuuRFr65FsOmv-NkZV4dAQ,519
+undergraves-0.1.0.dist-info/licenses/LICENSE,sha256=cbiXy2wY39x58fpk6gip2vOUcA9o7KGrtNd0EK39JM4,1106
+undergraves-0.1.0.dist-info/METADATA,sha256=xOwa2GSQUKnsCLs3Jg2uYq6bnWRoYsppQ-UulYYoO3s,1759
+undergraves-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+undergraves-0.1.0.dist-info/entry_points.txt,sha256=t7koYKTjGDZKLTWZu9jbF8PhyBhBv4E4LiKrdpcmv-w,52
+undergraves-0.1.0.dist-info/top_level.txt,sha256=SG-Lgb0oKMKTZyHCydXt_U6GMy0APVCmohgYFgMf11o,12
+undergraves-0.1.0.dist-info/RECORD,,

undergraves-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

undergraves-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+undergraves = undergraves.cli:run

undergraves-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,19 @@
+ Copyright (c) 2026 Gabriel Augusto da Costa Dill
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.

undergraves-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+undergraves