owlscan 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- owlscan/__init__.py +47 -0
- owlscan/cli.py +339 -0
- owlscan/core/__init__.py +0 -0
- owlscan/core/config.py +244 -0
- owlscan/core/database.py +83 -0
- owlscan/core/engine.py +276 -0
- owlscan/core/models.py +279 -0
- owlscan/exporters/__init__.py +0 -0
- owlscan/exporters/manager.py +528 -0
- owlscan/intel/__init__.py +0 -0
- owlscan/intel/apis/__init__.py +0 -0
- owlscan/intel/apis/all_apis.py +1558 -0
- owlscan/intel/apis/base.py +215 -0
- owlscan/intel/apis/shodan_api.py +165 -0
- owlscan/intel/orchestrator.py +130 -0
- owlscan/intel/people/__init__.py +0 -0
- owlscan/intel/people/aggregator.py +254 -0
- owlscan/scrapers/__init__.py +0 -0
- owlscan/scrapers/api_hunter.py +286 -0
- owlscan/scrapers/crawler.py +366 -0
- owlscan/scrapers/dns_recon.py +379 -0
- owlscan/scrapers/port_scanner.py +267 -0
- owlscan/scrapers/spiders/__init__.py +0 -0
- owlscan/scrapers/tech_detector.py +447 -0
- owlscan/web/__init__.py +0 -0
- owlscan/web/app.py +134 -0
- owlscan/web/routes/__init__.py +0 -0
- owlscan/web/routes/api.py +106 -0
- owlscan/web/routes/dashboard.py +48 -0
- owlscan/web/routes/export.py +66 -0
- owlscan/web/routes/intel.py +59 -0
- owlscan/web/routes/scans.py +124 -0
- owlscan/web/routes/settings.py +63 -0
- owlscan-1.2.0.dist-info/METADATA +440 -0
- owlscan-1.2.0.dist-info/RECORD +39 -0
- owlscan-1.2.0.dist-info/WHEEL +5 -0
- owlscan-1.2.0.dist-info/entry_points.txt +3 -0
- owlscan-1.2.0.dist-info/licenses/LICENSE +32 -0
- owlscan-1.2.0.dist-info/top_level.txt +1 -0
owlscan/__init__.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""
|
|
2
|
+
██████╗ ██╗ ██╗██╗ ███████╗ ██████╗ █████╗ ███╗ ██╗
|
|
3
|
+
██╔═══██╗██║ ██║██║ ██╔════╝██╔════╝██╔══██╗████╗ ██║
|
|
4
|
+
██║ ██║██║ █╗ ██║██║ ███████╗██║ ███████║██╔██╗ ██║
|
|
5
|
+
██║ ██║██║███╗██║██║ ╚════██║██║ ██╔══██║██║╚██╗██║
|
|
6
|
+
╚██████╔╝╚███╔███╔╝███████╗███████║╚██████╗██║ ██║██║ ╚████║
|
|
7
|
+
╚═════╝ ╚══╝╚══╝ ╚══════╝╚══════╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═══╝
|
|
8
|
+
|
|
9
|
+
>> OPEN-SOURCE OSINT INTELLIGENCE FRAMEWORK <<
|
|
10
|
+
>> PHANTOM SIGNAL // v1.2.0 <<
|
|
11
|
+
>> "See everything. Leave no trace." <<
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__version__ = "1.2.0"
|
|
15
|
+
__codename__ = "PHANTOM SIGNAL"
|
|
16
|
+
__author__ = "packetsn1ffer"
|
|
17
|
+
__credits__ = [
|
|
18
|
+
"packetsn1ffer — concept, design, threat modeling, module specification",
|
|
19
|
+
"Claude (Anthropic) — AI architecture, implementation, full-stack development",
|
|
20
|
+
]
|
|
21
|
+
__license__ = "MIT"
|
|
22
|
+
__url__ = "https://github.com/owlscan/owlscan"
|
|
23
|
+
|
|
24
|
+
BANNER = r"""
|
|
25
|
+
██████╗ ██╗ ██╗██╗ ███████╗ ██████╗ █████╗ ███╗ ██╗
|
|
26
|
+
██╔═══██╗██║ ██║██║ ██╔════╝██╔════╝██╔══██╗████╗ ██║
|
|
27
|
+
██║ ██║██║ █╗ ██║██║ ███████╗██║ ███████║██╔██╗ ██║
|
|
28
|
+
██║ ██║██║███╗██║██║ ╚════██║██║ ██╔══██║██║╚██╗██║
|
|
29
|
+
╚██████╔╝╚███╔███╔╝███████╗███████║╚██████╗██║ ██║██║ ╚████║
|
|
30
|
+
╚═════╝ ╚══╝╚══╝ ╚══════╝╚══════╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═══╝
|
|
31
|
+
|
|
32
|
+
░░░░░░ PHANTOM SIGNAL // v{version} // OSINT FRAMEWORK ░░░░░░
|
|
33
|
+
░ "The night sees all. The owl forgets nothing." ░
|
|
34
|
+
░ by packetsn1ffer // AI arch: Claude (Anthropic) ░
|
|
35
|
+
░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
|
|
36
|
+
""".format(version=__version__)
|
|
37
|
+
|
|
38
|
+
DISCLAIMER = """
|
|
39
|
+
╔══════════════════════════════════════════════════════════════════╗
|
|
40
|
+
║ ⚠ OPERATIVE WARNING ⚠ ║
|
|
41
|
+
║ OwlScan is for authorized security research, OSINT, and ║
|
|
42
|
+
║ educational purposes ONLY. You are solely responsible for ║
|
|
43
|
+
║ ensuring your use complies with all applicable laws. ║
|
|
44
|
+
║ Unauthorized use against systems you don't own is illegal. ║
|
|
45
|
+
║ The developers assume NO liability for misuse. ║
|
|
46
|
+
╚══════════════════════════════════════════════════════════════════╝
|
|
47
|
+
"""
|
owlscan/cli.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OwlScan CLI — Ghost Terminal Interface
|
|
3
|
+
Operative command-line control for the shadow grid.
|
|
4
|
+
|
|
5
|
+
Author: packetsn1ffer
|
|
6
|
+
AI: Claude (Anthropic)
|
|
7
|
+
License: MIT — see LICENSE
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
import click
|
|
18
|
+
from rich.console import Console
|
|
19
|
+
from rich.panel import Panel
|
|
20
|
+
from rich.table import Table
|
|
21
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
|
22
|
+
from rich.syntax import Syntax
|
|
23
|
+
from rich import print as rprint
|
|
24
|
+
|
|
25
|
+
from owlscan import __version__, BANNER, DISCLAIMER
|
|
26
|
+
|
|
27
|
+
console = Console(highlight=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def print_banner():
|
|
31
|
+
console.print(BANNER, style="bold green")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@click.group()
|
|
35
|
+
@click.version_option(__version__, prog_name="owlscan")
|
|
36
|
+
@click.option("--config", "-c", default=None, help="Path to config file")
|
|
37
|
+
@click.pass_context
|
|
38
|
+
def main(ctx, config):
|
|
39
|
+
"""
|
|
40
|
+
🦉 OwlScan — Open Source OSINT Intelligence Framework
|
|
41
|
+
|
|
42
|
+
\b
|
|
43
|
+
"See everything. Leave no trace."
|
|
44
|
+
"""
|
|
45
|
+
ctx.ensure_object(dict)
|
|
46
|
+
ctx.obj["config_path"] = config
|
|
47
|
+
|
|
48
|
+
from owlscan.core.config import OwlScanConfig
|
|
49
|
+
from owlscan.core.database import init_db
|
|
50
|
+
if config:
|
|
51
|
+
OwlScanConfig(config_path=config)
|
|
52
|
+
init_db()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@main.command()
|
|
56
|
+
@click.option("--host", "-H", default=None, help="Bind host (default: 127.0.0.1)")
|
|
57
|
+
@click.option("--port", "-p", default=None, type=int, help="Bind port (default: 5000)")
|
|
58
|
+
@click.option("--debug", is_flag=True, help="Enable debug mode")
|
|
59
|
+
@click.option("--open-browser", "-b", is_flag=True, help="Auto-open browser")
|
|
60
|
+
def web(host, port, debug, open_browser):
|
|
61
|
+
"""Launch the OwlScan web interface — the Shadow Grid control panel."""
|
|
62
|
+
print_banner()
|
|
63
|
+
console.print(DISCLAIMER, style="bold yellow")
|
|
64
|
+
|
|
65
|
+
from owlscan.core.config import config as cfg
|
|
66
|
+
_host = host or cfg.get("server", "host", default="127.0.0.1")
|
|
67
|
+
_port = port or cfg.get("server", "port", default=5000)
|
|
68
|
+
_debug = debug or cfg.get("server", "debug", default=False)
|
|
69
|
+
|
|
70
|
+
console.print(f"\n[bold green]>> SIGNAL LOCKED[/bold green]")
|
|
71
|
+
console.print(f" Grid interface: [bold cyan]http://{_host}:{_port}[/bold cyan]")
|
|
72
|
+
console.print(f" Mode: {'[yellow]DEBUG[/yellow]' if _debug else '[green]STEALTH[/green]'}")
|
|
73
|
+
console.print(f" [dim]Press Ctrl+C to sever the connection[/dim]\n")
|
|
74
|
+
|
|
75
|
+
if open_browser:
|
|
76
|
+
import threading, webbrowser
|
|
77
|
+
threading.Timer(1.5, lambda: webbrowser.open(f"http://{_host}:{_port}")).start()
|
|
78
|
+
|
|
79
|
+
from owlscan.web.app import create_app, socketio
|
|
80
|
+
|
|
81
|
+
app = create_app()
|
|
82
|
+
socketio.run(app, host=_host, port=_port, debug=_debug, use_reloader=False)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@main.command()
|
|
86
|
+
@click.argument("target")
|
|
87
|
+
@click.option("--type", "-t", "scan_type",
|
|
88
|
+
type=click.Choice(["web_recon", "ip_recon", "domain_recon", "people_intel", "full_spectrum"]),
|
|
89
|
+
default="web_recon", help="Scan type")
|
|
90
|
+
@click.option("--modules", "-m", multiple=True,
|
|
91
|
+
help="Modules to run (dns_recon, port_scan, tech_detect, api_hunt, web_crawl, intel)")
|
|
92
|
+
@click.option("--profile", "-p",
|
|
93
|
+
type=click.Choice(["quick", "standard", "deep", "ghost"]),
|
|
94
|
+
default="standard")
|
|
95
|
+
@click.option("--output", "-o", default=None, help="Output file path")
|
|
96
|
+
@click.option("--format", "-f", "fmt",
|
|
97
|
+
type=click.Choice(["json", "csv", "html", "xml", "pdf", "markdown", "stix"]),
|
|
98
|
+
default="json", help="Output format")
|
|
99
|
+
@click.option("--compress", is_flag=True)
|
|
100
|
+
@click.option("--encrypt", is_flag=True)
|
|
101
|
+
@click.option("--password", default=None, help="Encryption password")
|
|
102
|
+
@click.option("--no-robots", is_flag=True, help="Ignore robots.txt")
|
|
103
|
+
def scan(target, scan_type, modules, profile, output, fmt, compress, encrypt, password, no_robots):
|
|
104
|
+
"""Launch a ghost run against a target from the command line."""
|
|
105
|
+
print_banner()
|
|
106
|
+
console.print(DISCLAIMER, style="yellow")
|
|
107
|
+
|
|
108
|
+
console.print(f"\n[bold green]◈ TARGET LOCKED:[/bold green] [bold white]{target}[/bold white]")
|
|
109
|
+
console.print(f"[dim] Scan type: {scan_type} | Profile: {profile}[/dim]\n")
|
|
110
|
+
|
|
111
|
+
if not click.confirm("Confirm you have authorization to scan this target?", default=False):
|
|
112
|
+
console.print("[red]Mission aborted — no authorization confirmed.[/red]")
|
|
113
|
+
sys.exit(1)
|
|
114
|
+
|
|
115
|
+
from owlscan.core.config import config as cfg
|
|
116
|
+
from owlscan.core.database import get_db
|
|
117
|
+
from owlscan.core.models import Scan, ScanType, ScanStatus
|
|
118
|
+
from owlscan.core.engine import PhantomEngine
|
|
119
|
+
|
|
120
|
+
if no_robots:
|
|
121
|
+
cfg.set("scraper", "respect_robots_txt", value=False)
|
|
122
|
+
|
|
123
|
+
with get_db() as db:
|
|
124
|
+
scan_obj = Scan(
|
|
125
|
+
name=f"CLI Ghost Run — {target[:30]}",
|
|
126
|
+
target=target,
|
|
127
|
+
scan_type=ScanType(scan_type),
|
|
128
|
+
profile=profile,
|
|
129
|
+
modules_enabled=list(modules) if modules else ["dns_recon", "port_scan", "tech_detect", "api_hunt", "intel"],
|
|
130
|
+
options={"depth": 2 if profile == "quick" else 3},
|
|
131
|
+
)
|
|
132
|
+
db.add(scan_obj)
|
|
133
|
+
db.flush()
|
|
134
|
+
scan_id = scan_obj.id
|
|
135
|
+
|
|
136
|
+
engine = PhantomEngine()
|
|
137
|
+
|
|
138
|
+
with Progress(
|
|
139
|
+
SpinnerColumn(style="green"),
|
|
140
|
+
TextColumn("[bold green]{task.description}"),
|
|
141
|
+
BarColumn(bar_width=40, style="green", complete_style="bright_green"),
|
|
142
|
+
TextColumn("[dim]{task.percentage:.0f}%"),
|
|
143
|
+
console=console,
|
|
144
|
+
) as progress:
|
|
145
|
+
task = progress.add_task("Ghost run in progress...", total=100)
|
|
146
|
+
|
|
147
|
+
def _update_progress(p):
|
|
148
|
+
progress.update(task, completed=p)
|
|
149
|
+
|
|
150
|
+
async def _run():
|
|
151
|
+
engine._progress_callbacks[scan_id] = [lambda p, _: _update_progress(p)]
|
|
152
|
+
await engine.launch_scan(scan_id)
|
|
153
|
+
|
|
154
|
+
asyncio.run(_run())
|
|
155
|
+
|
|
156
|
+
with get_db() as db:
|
|
157
|
+
from owlscan.core.models import ScanResult
|
|
158
|
+
scan_obj = db.query(Scan).filter(Scan.id == scan_id).first()
|
|
159
|
+
results = db.query(ScanResult).filter(ScanResult.scan_id == scan_id).all()
|
|
160
|
+
scan_dict = scan_obj.to_dict()
|
|
161
|
+
results_list = [r.to_dict() for r in results]
|
|
162
|
+
|
|
163
|
+
# Display summary
|
|
164
|
+
table = Table(title="[bold green]GHOST RUN RESULTS[/bold green]", show_header=True, header_style="bold green")
|
|
165
|
+
table.add_column("MODULE", style="green")
|
|
166
|
+
table.add_column("TYPE", style="cyan")
|
|
167
|
+
table.add_column("SOURCE")
|
|
168
|
+
table.add_column("CONF", justify="right")
|
|
169
|
+
|
|
170
|
+
for r in results_list[:40]:
|
|
171
|
+
table.add_row(
|
|
172
|
+
r.get("module", "—"),
|
|
173
|
+
r.get("result_type", "—"),
|
|
174
|
+
r.get("source", "—"),
|
|
175
|
+
f"{(r.get('confidence', 0) * 100):.0f}%",
|
|
176
|
+
)
|
|
177
|
+
console.print(table)
|
|
178
|
+
|
|
179
|
+
console.print(f"\n[bold]Shadow Score:[/bold] [{'red' if scan_dict.get('shadow_score', 0) > 70 else 'green'}]{scan_dict.get('shadow_score', 0):.0f}/100[/]")
|
|
180
|
+
console.print(f"[bold]Threat Level:[/bold] {scan_dict.get('threat_level', 'unknown').upper()}")
|
|
181
|
+
console.print(f"[bold]Results:[/bold] {len(results_list)}")
|
|
182
|
+
|
|
183
|
+
if output:
|
|
184
|
+
from owlscan.exporters.manager import ExportManager
|
|
185
|
+
manager = ExportManager(output_dir=str(Path(output).parent))
|
|
186
|
+
result = manager.export(
|
|
187
|
+
scan_id=scan_id,
|
|
188
|
+
fmt=fmt,
|
|
189
|
+
compress=compress,
|
|
190
|
+
encrypt=encrypt,
|
|
191
|
+
encryption_password=password,
|
|
192
|
+
)
|
|
193
|
+
console.print(f"\n[bold green]✓ Intel packet exported:[/bold green] {result['file_path']}")
|
|
194
|
+
console.print(f" Size: {result['file_size_human']} | SHA256: {result['checksum_sha256'][:16]}...")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@main.command()
|
|
198
|
+
@click.option("--first-name", "-f", default=None)
|
|
199
|
+
@click.option("--last-name", "-l", default=None)
|
|
200
|
+
@click.option("--email", "-e", default=None)
|
|
201
|
+
@click.option("--phone", "-p", default=None)
|
|
202
|
+
@click.option("--username", "-u", default=None)
|
|
203
|
+
@click.option("--output", "-o", default=None)
|
|
204
|
+
def profile(first_name, last_name, email, phone, username, output):
|
|
205
|
+
"""Build a shadow profile — aggregate people intelligence from all configured APIs."""
|
|
206
|
+
print_banner()
|
|
207
|
+
|
|
208
|
+
if not any([first_name, last_name, email, phone, username]):
|
|
209
|
+
console.print("[red]At least one identifier required.[/red]")
|
|
210
|
+
sys.exit(1)
|
|
211
|
+
|
|
212
|
+
from owlscan.intel.people.aggregator import ShadowProfileBuilder
|
|
213
|
+
from owlscan.core.config import config as cfg
|
|
214
|
+
|
|
215
|
+
console.print(f"\n[bold cyan]◉ INITIATING SHADOW PROFILER...[/bold cyan]")
|
|
216
|
+
|
|
217
|
+
with console.status("[bold green]Scanning the grid...", spinner="dots"):
|
|
218
|
+
builder = ShadowProfileBuilder(cfg)
|
|
219
|
+
result = asyncio.run(builder.build_profile(
|
|
220
|
+
first_name=first_name,
|
|
221
|
+
last_name=last_name,
|
|
222
|
+
email=email,
|
|
223
|
+
phone=phone,
|
|
224
|
+
username=username,
|
|
225
|
+
))
|
|
226
|
+
|
|
227
|
+
console.print(f"\n[bold green]SHADOW PROFILE COMPILED[/bold green]")
|
|
228
|
+
console.print(f"Confidence: [cyan]{result.get('confidence', 0):.0%}[/cyan]")
|
|
229
|
+
console.print(f"Shadow Score: [{'red' if result.get('shadow_score', 0) > 60 else 'green'}]{result.get('shadow_score', 0):.0f}/100[/]")
|
|
230
|
+
console.print(f"Sources: [cyan]{', '.join(result.get('sources', []))}[/cyan]")
|
|
231
|
+
|
|
232
|
+
if result.get("emails"):
|
|
233
|
+
console.print(f"\n[bold]Emails:[/bold]")
|
|
234
|
+
for e in result["emails"][:10]:
|
|
235
|
+
console.print(f" ● {e.get('value', e)}")
|
|
236
|
+
|
|
237
|
+
if result.get("phones"):
|
|
238
|
+
console.print(f"\n[bold]Phones:[/bold]")
|
|
239
|
+
for p in result["phones"][:10]:
|
|
240
|
+
console.print(f" ● {p.get('value', p)}")
|
|
241
|
+
|
|
242
|
+
if result.get("addresses"):
|
|
243
|
+
console.print(f"\n[bold]Addresses:[/bold]")
|
|
244
|
+
for a in result["addresses"][:5]:
|
|
245
|
+
console.print(f" ● {json.dumps(a, default=str)[:120]}")
|
|
246
|
+
|
|
247
|
+
if result.get("breach_data"):
|
|
248
|
+
console.print(f"\n[bold red]⚠ BREACHES DETECTED: {len(result['breach_data'])}[/bold red]")
|
|
249
|
+
for b in result["breach_data"][:5]:
|
|
250
|
+
console.print(f" ✗ {b.get('name', '?')} ({b.get('breach_date', '?')})")
|
|
251
|
+
|
|
252
|
+
if output:
|
|
253
|
+
with open(output, "w") as f:
|
|
254
|
+
json.dump(result, f, indent=2, default=str)
|
|
255
|
+
console.print(f"\n[green]✓ Profile saved: {output}[/green]")
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@main.command()
|
|
259
|
+
def status():
|
|
260
|
+
"""Show the OwlScan grid status — configured APIs, recent scans."""
|
|
261
|
+
print_banner()
|
|
262
|
+
|
|
263
|
+
from owlscan.core.database import get_db
|
|
264
|
+
from owlscan.core.models import Scan, ScanStatus
|
|
265
|
+
from owlscan.intel.orchestrator import IntelOrchestrator
|
|
266
|
+
from owlscan.core.config import config as cfg
|
|
267
|
+
|
|
268
|
+
with get_db() as db:
|
|
269
|
+
total = db.query(Scan).count()
|
|
270
|
+
running = db.query(Scan).filter(Scan.status == ScanStatus.RUNNING).count()
|
|
271
|
+
recent = db.query(Scan).order_by(Scan.created_at.desc()).limit(5).all()
|
|
272
|
+
|
|
273
|
+
orch = IntelOrchestrator(cfg)
|
|
274
|
+
apis = orch.get_api_status()
|
|
275
|
+
configured = [a for a in apis if a.get("is_configured")]
|
|
276
|
+
|
|
277
|
+
table = Table(title="[bold green]GRID STATUS[/bold green]", show_header=True, header_style="bold green")
|
|
278
|
+
table.add_column("METRIC")
|
|
279
|
+
table.add_column("VALUE", style="cyan")
|
|
280
|
+
table.add_row("Total Scans", str(total))
|
|
281
|
+
table.add_row("Active Ghosts", str(running))
|
|
282
|
+
table.add_row("APIs Online", f"{len(configured)}/{len(apis)}")
|
|
283
|
+
table.add_row("Version", __version__)
|
|
284
|
+
console.print(table)
|
|
285
|
+
|
|
286
|
+
api_table = Table(title="[bold cyan]API ARSENAL[/bold cyan]", show_header=True, header_style="bold cyan")
|
|
287
|
+
api_table.add_column("API")
|
|
288
|
+
api_table.add_column("STATUS")
|
|
289
|
+
api_table.add_column("TIER")
|
|
290
|
+
for api in sorted(apis, key=lambda x: (not x.get("is_configured"), x["name"])):
|
|
291
|
+
status_str = "[green]● ONLINE[/green]" if api.get("is_configured") else "[dim]○ OFFLINE[/dim]"
|
|
292
|
+
api_table.add_row(api["name"], status_str, api.get("tier", "?"))
|
|
293
|
+
console.print(api_table)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@main.command()
|
|
297
|
+
@click.argument("scan_id")
|
|
298
|
+
@click.option("--format", "-f", "fmt", default="json",
|
|
299
|
+
type=click.Choice(["json", "csv", "html", "xml", "pdf", "markdown", "stix"]))
|
|
300
|
+
@click.option("--output", "-o", default="./exports")
|
|
301
|
+
@click.option("--compress", is_flag=True)
|
|
302
|
+
@click.option("--encrypt", is_flag=True)
|
|
303
|
+
@click.option("--password", default=None)
|
|
304
|
+
def export(scan_id, fmt, output, compress, encrypt, password):
|
|
305
|
+
"""Export a ghost run's intel packet to a file."""
|
|
306
|
+
from owlscan.exporters.manager import ExportManager
|
|
307
|
+
manager = ExportManager(output_dir=output)
|
|
308
|
+
try:
|
|
309
|
+
result = manager.export(
|
|
310
|
+
scan_id=scan_id,
|
|
311
|
+
fmt=fmt,
|
|
312
|
+
compress=compress,
|
|
313
|
+
encrypt=encrypt,
|
|
314
|
+
encryption_password=password,
|
|
315
|
+
)
|
|
316
|
+
console.print(f"[bold green]✓ Intel packet compiled:[/bold green]")
|
|
317
|
+
console.print(f" File: {result['file_path']}")
|
|
318
|
+
console.print(f" Size: {result['file_size_human']}")
|
|
319
|
+
console.print(f" Results: {result['result_count']}")
|
|
320
|
+
console.print(f" SHA256: {result['checksum_sha256']}")
|
|
321
|
+
except Exception as e:
|
|
322
|
+
console.print(f"[red]Export failed: {e}[/red]")
|
|
323
|
+
sys.exit(1)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
@main.command()
|
|
327
|
+
def init():
|
|
328
|
+
"""Initialize OwlScan — create default config and database."""
|
|
329
|
+
print_banner()
|
|
330
|
+
from owlscan.core.database import init_db
|
|
331
|
+
init_db()
|
|
332
|
+
console.print("[bold green]✓ Grid initialized.[/bold green]")
|
|
333
|
+
console.print(" Config: ~/.owlscan/config.yaml")
|
|
334
|
+
console.print(" Database: owlscan.db")
|
|
335
|
+
console.print("\n[cyan]Next:[/cyan] Add API keys with: [bold]owlscan web[/bold] → Settings → Ghost Keys")
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
if __name__ == "__main__":
|
|
339
|
+
main()
|
owlscan/core/__init__.py
ADDED
|
File without changes
|
owlscan/core/config.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OwlScan Configuration Manager — Ghost Keys & Grid Settings
|
|
3
|
+
|
|
4
|
+
Author: packetsn1ffer
|
|
5
|
+
AI: Claude (Anthropic)
|
|
6
|
+
License: MIT — see LICENSE
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Dict, Optional
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
from dotenv import load_dotenv
|
|
16
|
+
|
|
17
|
+
load_dotenv()
|
|
18
|
+
|
|
19
|
+
_DEFAULT_CONFIG_PATH = Path(__file__).parent.parent.parent / "config" / "owlscan.yaml"
|
|
20
|
+
_USER_CONFIG_PATH = Path.home() / ".owlscan" / "config.yaml"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class OwlScanConfig:
|
|
24
|
+
"""Central configuration hub for the OwlScan grid."""
|
|
25
|
+
|
|
26
|
+
_instance: Optional["OwlScanConfig"] = None
|
|
27
|
+
|
|
28
|
+
def __new__(cls, config_path: Optional[str] = None) -> "OwlScanConfig":
|
|
29
|
+
if cls._instance is None:
|
|
30
|
+
cls._instance = super().__new__(cls)
|
|
31
|
+
cls._instance._initialized = False
|
|
32
|
+
return cls._instance
|
|
33
|
+
|
|
34
|
+
def __init__(self, config_path: Optional[str] = None) -> None:
|
|
35
|
+
if self._initialized:
|
|
36
|
+
return
|
|
37
|
+
self._initialized = True
|
|
38
|
+
self._config: Dict[str, Any] = {}
|
|
39
|
+
self._load_defaults()
|
|
40
|
+
self._load_file(_DEFAULT_CONFIG_PATH)
|
|
41
|
+
if _USER_CONFIG_PATH.exists():
|
|
42
|
+
self._load_file(_USER_CONFIG_PATH)
|
|
43
|
+
if config_path:
|
|
44
|
+
self._load_file(Path(config_path))
|
|
45
|
+
self._apply_env_overrides()
|
|
46
|
+
|
|
47
|
+
def _load_defaults(self) -> None:
|
|
48
|
+
self._config = {
|
|
49
|
+
"server": {
|
|
50
|
+
"host": "127.0.0.1",
|
|
51
|
+
"port": 5000,
|
|
52
|
+
"debug": False,
|
|
53
|
+
"secret_key": os.urandom(32).hex(),
|
|
54
|
+
"workers": 4,
|
|
55
|
+
},
|
|
56
|
+
"database": {
|
|
57
|
+
"url": "sqlite:///owlscan.db",
|
|
58
|
+
"pool_size": 10,
|
|
59
|
+
"echo": False,
|
|
60
|
+
},
|
|
61
|
+
"scraper": {
|
|
62
|
+
"concurrent_requests": 16,
|
|
63
|
+
"download_delay": 1.0,
|
|
64
|
+
"randomize_delay": True,
|
|
65
|
+
"respect_robots_txt": True,
|
|
66
|
+
"user_agent_rotation": True,
|
|
67
|
+
"follow_redirects": True,
|
|
68
|
+
"max_depth": 3,
|
|
69
|
+
"timeout": 30,
|
|
70
|
+
"javascript_rendering": False,
|
|
71
|
+
"proxy": None,
|
|
72
|
+
"tor_enabled": False,
|
|
73
|
+
"tor_port": 9050,
|
|
74
|
+
},
|
|
75
|
+
"port_scanner": {
|
|
76
|
+
"default_ports": [
|
|
77
|
+
21, 22, 23, 25, 53, 80, 110, 143, 443, 445,
|
|
78
|
+
993, 995, 1433, 1521, 3306, 3389, 5432, 5900,
|
|
79
|
+
6379, 8080, 8443, 8888, 9200, 27017,
|
|
80
|
+
],
|
|
81
|
+
"max_concurrent": 500,
|
|
82
|
+
"timeout": 3,
|
|
83
|
+
"service_detection": True,
|
|
84
|
+
},
|
|
85
|
+
"intel": {
|
|
86
|
+
"enabled_apis": [],
|
|
87
|
+
"rate_limit_buffer": 0.5,
|
|
88
|
+
"cache_ttl": 3600,
|
|
89
|
+
"auto_correlate": True,
|
|
90
|
+
"shadow_scoring": True,
|
|
91
|
+
},
|
|
92
|
+
"export": {
|
|
93
|
+
"default_format": "json",
|
|
94
|
+
"output_dir": "./exports",
|
|
95
|
+
"compression": False,
|
|
96
|
+
"encryption": False,
|
|
97
|
+
"encryption_algorithm": "AES-256-GCM",
|
|
98
|
+
},
|
|
99
|
+
"ghost_mode": {
|
|
100
|
+
"enabled": False,
|
|
101
|
+
"rotate_identity": True,
|
|
102
|
+
"header_spoofing": True,
|
|
103
|
+
"delay_jitter": True,
|
|
104
|
+
"jitter_range": [0.5, 3.0],
|
|
105
|
+
},
|
|
106
|
+
"neural_profiler": {
|
|
107
|
+
"enabled": False,
|
|
108
|
+
"model": "local",
|
|
109
|
+
"confidence_threshold": 0.75,
|
|
110
|
+
},
|
|
111
|
+
"notifications": {
|
|
112
|
+
"desktop": True,
|
|
113
|
+
"webhook_url": None,
|
|
114
|
+
"slack_token": None,
|
|
115
|
+
},
|
|
116
|
+
"api_keys": {},
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
def _load_file(self, path: Path) -> None:
|
|
120
|
+
if not path.exists():
|
|
121
|
+
return
|
|
122
|
+
with open(path) as f:
|
|
123
|
+
data = yaml.safe_load(f) or {}
|
|
124
|
+
self._deep_merge(self._config, data)
|
|
125
|
+
|
|
126
|
+
def _deep_merge(self, base: dict, override: dict) -> None:
|
|
127
|
+
for key, value in override.items():
|
|
128
|
+
if key in base and isinstance(base[key], dict) and isinstance(value, dict):
|
|
129
|
+
self._deep_merge(base[key], value)
|
|
130
|
+
else:
|
|
131
|
+
base[key] = value
|
|
132
|
+
|
|
133
|
+
def _apply_env_overrides(self) -> None:
|
|
134
|
+
env_map = {
|
|
135
|
+
"OWLSCAN_HOST": ("server", "host"),
|
|
136
|
+
"OWLSCAN_PORT": ("server", "port"),
|
|
137
|
+
"OWLSCAN_DEBUG": ("server", "debug"),
|
|
138
|
+
"OWLSCAN_DB_URL": ("database", "url"),
|
|
139
|
+
"OWLSCAN_SECRET_KEY": ("server", "secret_key"),
|
|
140
|
+
"OWLSCAN_TOR_ENABLED": ("scraper", "tor_enabled"),
|
|
141
|
+
"OWLSCAN_PROXY": ("scraper", "proxy"),
|
|
142
|
+
}
|
|
143
|
+
api_key_envs = {
|
|
144
|
+
"SHODAN_API_KEY": "shodan",
|
|
145
|
+
"CENSYS_API_ID": "censys_id",
|
|
146
|
+
"CENSYS_API_SECRET": "censys_secret",
|
|
147
|
+
"HUNTER_API_KEY": "hunter",
|
|
148
|
+
"HIBP_API_KEY": "hibp",
|
|
149
|
+
"VIRUSTOTAL_API_KEY": "virustotal",
|
|
150
|
+
"ABUSEIPDB_API_KEY": "abuseipdb",
|
|
151
|
+
"GREYNOISE_API_KEY": "greynoise",
|
|
152
|
+
"IPINFO_TOKEN": "ipinfo",
|
|
153
|
+
"SECURITYTRAILS_API_KEY": "securitytrails",
|
|
154
|
+
"URLSCAN_API_KEY": "urlscan",
|
|
155
|
+
"ALIENVAULT_API_KEY": "alienvault",
|
|
156
|
+
"GITHUB_TOKEN": "github",
|
|
157
|
+
"TWITTER_BEARER_TOKEN": "twitter_bearer",
|
|
158
|
+
"ZOOMEYE_API_KEY": "zoomeye",
|
|
159
|
+
"FULLCONTACT_API_KEY": "fullcontact",
|
|
160
|
+
"PIPL_API_KEY": "pipl",
|
|
161
|
+
"BINARYEDGE_API_KEY": "binaryedge",
|
|
162
|
+
"SPYSE_API_KEY": "spyse",
|
|
163
|
+
"RISKIQ_USERNAME": "riskiq_user",
|
|
164
|
+
"RISKIQ_KEY": "riskiq_key",
|
|
165
|
+
"WHOISXML_API_KEY": "whoisxml",
|
|
166
|
+
"GOOGLE_CSE_KEY": "google_cse",
|
|
167
|
+
"GOOGLE_CSE_ID": "google_cse_id",
|
|
168
|
+
"BING_SEARCH_KEY": "bing",
|
|
169
|
+
"SPOKEO_API_KEY": "spokeo",
|
|
170
|
+
"WHITEPAGES_API_KEY": "whitepages",
|
|
171
|
+
"INTELIUS_API_KEY": "intelius",
|
|
172
|
+
"CLEARBIT_API_KEY": "clearbit",
|
|
173
|
+
"TELEGRAM_BOT_TOKEN": "telegram",
|
|
174
|
+
}
|
|
175
|
+
for env_var, (section, key) in env_map.items():
|
|
176
|
+
val = os.getenv(env_var)
|
|
177
|
+
if val is not None:
|
|
178
|
+
if key in ("port",):
|
|
179
|
+
val = int(val)
|
|
180
|
+
elif val.lower() in ("true", "false"):
|
|
181
|
+
val = val.lower() == "true"
|
|
182
|
+
self._config[section][key] = val
|
|
183
|
+
|
|
184
|
+
for env_var, api_name in api_key_envs.items():
|
|
185
|
+
val = os.getenv(env_var)
|
|
186
|
+
if val:
|
|
187
|
+
self._config["api_keys"][api_name] = val
|
|
188
|
+
|
|
189
|
+
def get(self, *keys: str, default: Any = None) -> Any:
|
|
190
|
+
node = self._config
|
|
191
|
+
for k in keys:
|
|
192
|
+
if not isinstance(node, dict):
|
|
193
|
+
return default
|
|
194
|
+
node = node.get(k, default)
|
|
195
|
+
return node
|
|
196
|
+
|
|
197
|
+
def set(self, *keys: str, value: Any) -> None:
|
|
198
|
+
node = self._config
|
|
199
|
+
for k in keys[:-1]:
|
|
200
|
+
node = node.setdefault(k, {})
|
|
201
|
+
node[keys[-1]] = value
|
|
202
|
+
|
|
203
|
+
def get_api_key(self, service: str) -> Optional[str]:
|
|
204
|
+
return self._config["api_keys"].get(service)
|
|
205
|
+
|
|
206
|
+
def set_api_key(self, service: str, key: str) -> None:
|
|
207
|
+
self._config["api_keys"][service] = key
|
|
208
|
+
self._persist_user_config()
|
|
209
|
+
|
|
210
|
+
def _persist_user_config(self) -> None:
|
|
211
|
+
_USER_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
with open(_USER_CONFIG_PATH, "w") as f:
|
|
213
|
+
yaml.dump({"api_keys": self._config["api_keys"]}, f)
|
|
214
|
+
|
|
215
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
216
|
+
import copy
|
|
217
|
+
d = copy.deepcopy(self._config)
|
|
218
|
+
for k in d.get("api_keys", {}):
|
|
219
|
+
if d["api_keys"][k]:
|
|
220
|
+
d["api_keys"][k] = "***REDACTED***"
|
|
221
|
+
return d
|
|
222
|
+
|
|
223
|
+
@property
|
|
224
|
+
def server(self) -> Dict:
|
|
225
|
+
return self._config["server"]
|
|
226
|
+
|
|
227
|
+
@property
|
|
228
|
+
def scraper(self) -> Dict:
|
|
229
|
+
return self._config["scraper"]
|
|
230
|
+
|
|
231
|
+
@property
|
|
232
|
+
def intel(self) -> Dict:
|
|
233
|
+
return self._config["intel"]
|
|
234
|
+
|
|
235
|
+
@property
|
|
236
|
+
def export(self) -> Dict:
|
|
237
|
+
return self._config["export"]
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def ghost_mode(self) -> Dict:
|
|
241
|
+
return self._config["ghost_mode"]
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
config = OwlScanConfig()
|