vulpfest 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulpfest/__init__.py +1 -0
- vulpfest/__main__.py +108 -0
- vulpfest/banner.py +69 -0
- vulpfest/logo.py +19 -0
- vulpfest/tools/gmapscout/cli.py +51 -0
- vulpfest/tools/gmapscout/gmapscout_core.py +202 -0
- vulpfest/tools/leadscout/cli.py +52 -0
- vulpfest/tools/leadscout/leadscout_core.py +72 -0
- vulpfest/tools/scamcheck/checker.py +266 -0
- vulpfest/tools/scamcheck/cli.py +88 -0
- vulpfest/tools/scamcheck/ssrf_guard.py +142 -0
- vulpfest/tools/scanner/auth_scan.py +175 -0
- vulpfest/tools/scanner/cli.py +167 -0
- vulpfest/tools/scanner/deep_web.py +353 -0
- vulpfest/tools/scanner/dns_email.py +202 -0
- vulpfest/tools/scanner/manual_review.py +60 -0
- vulpfest/tools/scanner/repo_scan.py +434 -0
- vulpfest/tools/scanner/scanner.py +886 -0
- vulpfest/tools/scanner/ssrf_guard.py +142 -0
- vulpfest/ui.py +38 -0
- vulpfest-1.0.0.dist-info/METADATA +103 -0
- vulpfest-1.0.0.dist-info/RECORD +25 -0
- vulpfest-1.0.0.dist-info/WHEEL +4 -0
- vulpfest-1.0.0.dist-info/entry_points.txt +2 -0
- vulpfest-1.0.0.dist-info/licenses/LICENSE +21 -0
vulpfest/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
vulpfest/__main__.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Perintah `vulpfest`: menu interaktif + menjalankan keempat tools Vulpfest Lab.
|
|
2
|
+
|
|
3
|
+
vulpfest menu: pilih tool dengan nomor
|
|
4
|
+
vulpfest scan example.com Cyber Web Scanner
|
|
5
|
+
vulpfest check suspicious.link Scam Check
|
|
6
|
+
vulpfest leads --roles ... --industries ... LeadScout
|
|
7
|
+
vulpfest gmaps "coffee shop jakarta" GMaps Scout
|
|
8
|
+
"""
|
|
9
|
+
import importlib.util
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import __version__
|
|
14
|
+
from .banner import render
|
|
15
|
+
|
|
16
|
+
TOOLS_DIR = Path(__file__).resolve().parent / "tools"
|
|
17
|
+
TOOLS = {"scan": "scanner", "check": "scamcheck", "leads": "leadscout", "gmaps": "gmapscout"}
|
|
18
|
+
ORDER = ["scan", "check", "leads", "gmaps"]
|
|
19
|
+
NEEDS = {
|
|
20
|
+
"leads": [("ddgs", "duckduckgo_search"), ("pandas",)],
|
|
21
|
+
"gmaps": [("selenium",), ("webdriver_manager",), ("pandas",)],
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _has(names):
|
|
26
|
+
return any(importlib.util.find_spec(n) is not None for n in names)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def available():
|
|
30
|
+
return {k for k, groups in NEEDS.items() if all(_has(g) for g in groups)}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _utf8():
|
|
34
|
+
for stream in (sys.stdout, sys.stderr):
|
|
35
|
+
try:
|
|
36
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
37
|
+
except Exception:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def run_tool(key, args):
|
|
42
|
+
if key in NEEDS and key not in available():
|
|
43
|
+
print(f'{key} needs extra packages: pip install "vulpfest[{key}]"', file=sys.stderr)
|
|
44
|
+
if key == "gmaps":
|
|
45
|
+
print("GMaps Scout also needs Google Chrome installed.", file=sys.stderr)
|
|
46
|
+
return 2
|
|
47
|
+
tool_dir = TOOLS_DIR / TOOLS[key]
|
|
48
|
+
sys.path.insert(0, str(tool_dir))
|
|
49
|
+
sys.argv = [f"vulpfest {key}"] + list(args)
|
|
50
|
+
spec = importlib.util.spec_from_file_location(f"vulpfest_{TOOLS[key]}_cli", tool_dir / "cli.py")
|
|
51
|
+
module = importlib.util.module_from_spec(spec)
|
|
52
|
+
try:
|
|
53
|
+
spec.loader.exec_module(module)
|
|
54
|
+
rc = module.main()
|
|
55
|
+
except SystemExit as e:
|
|
56
|
+
if e.code is None:
|
|
57
|
+
return 0
|
|
58
|
+
return e.code if isinstance(e.code, int) else 1
|
|
59
|
+
return rc if isinstance(rc, int) else 0
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def interactive():
|
|
63
|
+
print(render(available()))
|
|
64
|
+
try:
|
|
65
|
+
choice = input("\nPilih tool (1-4), Enter untuk keluar: ").strip().lower()
|
|
66
|
+
if not choice:
|
|
67
|
+
return 0
|
|
68
|
+
key = ORDER[int(choice) - 1] if choice.isdigit() and 1 <= int(choice) <= 4 else choice
|
|
69
|
+
if key not in TOOLS:
|
|
70
|
+
print("Pilihan tidak dikenal.")
|
|
71
|
+
return 2
|
|
72
|
+
if key == "scan":
|
|
73
|
+
args = [input("Website (mis. example.com): ").strip()]
|
|
74
|
+
elif key == "check":
|
|
75
|
+
args = [input("Tautan yang ingin diperiksa: ").strip()]
|
|
76
|
+
elif key == "leads":
|
|
77
|
+
args = ["--roles", input("Roles, pisahkan dengan koma: ").strip(),
|
|
78
|
+
"--industries", input("Industries, pisahkan dengan koma: ").strip()]
|
|
79
|
+
else:
|
|
80
|
+
args = [input("Kata kunci pencarian (mis. coffee shop jakarta): ").strip()]
|
|
81
|
+
except (EOFError, KeyboardInterrupt):
|
|
82
|
+
print()
|
|
83
|
+
return 0
|
|
84
|
+
print()
|
|
85
|
+
return run_tool(key, args)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main(argv=None):
|
|
89
|
+
_utf8()
|
|
90
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
91
|
+
if not argv:
|
|
92
|
+
return interactive() if sys.stdin.isatty() else (print(render(available())) or 0)
|
|
93
|
+
head = argv[0].lower()
|
|
94
|
+
if head in ("-h", "--help", "help", "menu"):
|
|
95
|
+
print(render(available()))
|
|
96
|
+
return 0
|
|
97
|
+
if head in ("-v", "--version", "version"):
|
|
98
|
+
print(f"vulpfest {__version__}")
|
|
99
|
+
return 0
|
|
100
|
+
if head not in TOOLS:
|
|
101
|
+
print(f"Tool tidak dikenal: {argv[0]}\n")
|
|
102
|
+
print(render(available()))
|
|
103
|
+
return 2
|
|
104
|
+
return run_tool(head, argv[1:])
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__":
|
|
108
|
+
sys.exit(main())
|
vulpfest/banner.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Logo rubah Vulpfest (ASCII) dan menu, gaya neofetch: logo di kiri, info di kanan."""
|
|
2
|
+
from . import __version__
|
|
3
|
+
from .ui import c
|
|
4
|
+
|
|
5
|
+
# Logo Vulpfest: dibuat dari path SVG asli (telinga Sigma, V, percikan) oleh tools/make_logo.py.
|
|
6
|
+
from .logo import LOGO
|
|
7
|
+
from .ui import rgb
|
|
8
|
+
|
|
9
|
+
SAND, ROSE = (243, 222, 186), (232, 165, 152) # gradasi warna logo
|
|
10
|
+
_W = len(LOGO[0])
|
|
11
|
+
FOX = [l.ljust(_W) for l in LOGO + ["", "V U L P F E S T".center(_W)]]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def gradient(line):
|
|
15
|
+
n = max(len(line) - 1, 1)
|
|
16
|
+
out = []
|
|
17
|
+
for i, ch in enumerate(line):
|
|
18
|
+
if ch == " ":
|
|
19
|
+
out.append(ch)
|
|
20
|
+
continue
|
|
21
|
+
k = i / n
|
|
22
|
+
out.append(rgb(ch, *(round(SAND[j] + (ROSE[j] - SAND[j]) * k) for j in range(3))))
|
|
23
|
+
return "".join(out)
|
|
24
|
+
|
|
25
|
+
TOOLS = [
|
|
26
|
+
("scan", "Cyber Web Scanner", "Periksa keamanan situs Anda (header, SSL, file bocor, DNS).", "ready"),
|
|
27
|
+
("check", "Scam Check", "Periksa tautan mencurigakan sebelum diklik.", "ready"),
|
|
28
|
+
("leads", "LeadScout", "Riset prospek B2B dari hasil pencarian publik.", "leads"),
|
|
29
|
+
("gmaps", "GMaps Scout", "Kumpulkan detail bisnis publik ke spreadsheet.", "gmaps"),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
STATUS = {
|
|
33
|
+
"ready": c("siap", "green"),
|
|
34
|
+
"leads": c("perlu: pip install vulpfest[leads]", "yellow"),
|
|
35
|
+
"gmaps": c("perlu: pip install vulpfest[gmaps] + Chrome", "yellow"),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def render(available):
|
|
40
|
+
"""available: set berisi kunci tool yang dependensinya sudah terpasang."""
|
|
41
|
+
info = []
|
|
42
|
+
info.append(c("Vulpfest Lab", "fox", "bold") + c(f" v{__version__}", "gray"))
|
|
43
|
+
info.append(c("─" * 34, "gray"))
|
|
44
|
+
info.append(c("Alat gratis untuk keamanan & anti-penipuan.", "white"))
|
|
45
|
+
info.append("")
|
|
46
|
+
info.append(c("Tools:", "bold"))
|
|
47
|
+
for n, (key, name, desc, need) in enumerate(TOOLS, 1):
|
|
48
|
+
ok = need == "ready" or key in available
|
|
49
|
+
dot = c("●", "green") if ok else c("○", "yellow")
|
|
50
|
+
line = f" {dot} {c(str(n) + '.', 'bold')} {c(name, 'bold', 'white')} " + c(f"(vulpfest {key})", "cyan")
|
|
51
|
+
info.append(line)
|
|
52
|
+
info.append(" " + c(desc, "gray"))
|
|
53
|
+
if not ok:
|
|
54
|
+
info.append(" " + STATUS[need])
|
|
55
|
+
info.append("")
|
|
56
|
+
info.append(c("Contoh:", "bold") + c(" vulpfest scan example.com", "cyan"))
|
|
57
|
+
info.append(c(" vulpfest check bca-co-id.xyz", "cyan"))
|
|
58
|
+
info.append(c("Bantuan:", "bold") + c(" vulpfest <tool> --help", "cyan"))
|
|
59
|
+
|
|
60
|
+
fox = [gradient(line) for line in FOX]
|
|
61
|
+
# Sejajarkan logo (kiri) dan info (kanan)
|
|
62
|
+
pad = len(FOX)
|
|
63
|
+
rows = max(len(fox), len(info))
|
|
64
|
+
out = []
|
|
65
|
+
for i in range(rows):
|
|
66
|
+
left = fox[i] if i < len(fox) else " " * len(FOX[0])
|
|
67
|
+
right = info[i] if i < len(info) else ""
|
|
68
|
+
out.append(f"{left} {right}")
|
|
69
|
+
return "\n".join(out)
|
vulpfest/logo.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Dibuat otomatis oleh tools/make_logo.py dari path SVG logo Vulpfest."""
|
|
2
|
+
LOGO = [
|
|
3
|
+
' ▄▄ ▄▄ ',
|
|
4
|
+
' ████▄ ▄████ ',
|
|
5
|
+
' ████▀██▄ ▄██▀████ ',
|
|
6
|
+
' ████ ▀██▄▄ ▄▄██▀ ████ ',
|
|
7
|
+
' ▀███ ▀▀██▄ ██ ▄██▀▀ ███▀ ',
|
|
8
|
+
' ███▄ ▀███▄▀▀▄███▀ ▄███ ',
|
|
9
|
+
' ███ ▀██████▀ ███ ',
|
|
10
|
+
' ███ ▀▀ ███ ',
|
|
11
|
+
' ▀██▄ ▄ ▄ ▄██▀ ',
|
|
12
|
+
' ██████▄ ▄██████ ',
|
|
13
|
+
' ▀█▀██▄ ▄████▀ ',
|
|
14
|
+
' ▀███ ███▀ ',
|
|
15
|
+
' ▀███▄▄███▀ ',
|
|
16
|
+
' ██████ ',
|
|
17
|
+
' ▀██▀ ',
|
|
18
|
+
' ▀▀ ',
|
|
19
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""GMaps Scout from the command line.
|
|
2
|
+
|
|
3
|
+
Example:
|
|
4
|
+
python cli.py "coffee shop in Bali" --limit 20 --out leads.xlsx
|
|
5
|
+
"""
|
|
6
|
+
import argparse
|
|
7
|
+
import csv
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from gmapscout_core import COLUMNS, scrape
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main():
|
|
15
|
+
p = argparse.ArgumentParser(description="Turn publicly listed business details into a spreadsheet (Vulpfest Lab).")
|
|
16
|
+
p.add_argument("query", help='what to search for on Google Maps, e.g. "coffee shop in Bali"')
|
|
17
|
+
p.add_argument("--limit", type=int, default=20, help="maximum number of places (default 20, up to 200)")
|
|
18
|
+
p.add_argument("--out", metavar="FILE", help="save as .xlsx or .csv")
|
|
19
|
+
p.add_argument("--show-browser", action="store_true", help="show the browser window instead of running headless")
|
|
20
|
+
a = p.parse_args()
|
|
21
|
+
print("Reminder: collect only publicly listed business information and follow the terms of the sites you use.", file=sys.stderr)
|
|
22
|
+
|
|
23
|
+
def log(msg, type="info"):
|
|
24
|
+
print(f"[{time.strftime('%H:%M:%S')}] {type.upper():<7} {msg.replace('**', '')}", file=sys.stderr)
|
|
25
|
+
|
|
26
|
+
def progress(phase, current, total):
|
|
27
|
+
if phase == "extract":
|
|
28
|
+
print(f" reading {current}/{total}", end="\r", file=sys.stderr)
|
|
29
|
+
|
|
30
|
+
rows = scrape(a.query, max(1, min(a.limit, 200)), headless=not a.show_browser, log=log, on_progress=progress)
|
|
31
|
+
if not rows:
|
|
32
|
+
print("No data retrieved.", file=sys.stderr)
|
|
33
|
+
return 1
|
|
34
|
+
if a.out and a.out.lower().endswith(".xlsx"):
|
|
35
|
+
import pandas as pd
|
|
36
|
+
pd.DataFrame(rows, columns=COLUMNS).to_excel(a.out, index=False, sheet_name="Leads")
|
|
37
|
+
elif a.out:
|
|
38
|
+
with open(a.out, "w", newline="", encoding="utf-8-sig") as f:
|
|
39
|
+
w = csv.DictWriter(f, fieldnames=COLUMNS)
|
|
40
|
+
w.writeheader()
|
|
41
|
+
w.writerows(rows)
|
|
42
|
+
else:
|
|
43
|
+
for r in rows:
|
|
44
|
+
print(f"{r['Entity Name']}\t{r['Contact Number']}\t{r['Website URL']}\t{r['Address']}")
|
|
45
|
+
if a.out:
|
|
46
|
+
print(f"Saved {len(rows)} places to {a.out}", file=sys.stderr)
|
|
47
|
+
return 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
if __name__ == "__main__":
|
|
51
|
+
sys.exit(main())
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""GMaps Scout core: the scraping logic shared by the Streamlit app (app_scraper.py) and the command line (cli.py)."""
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
from selenium import webdriver
|
|
6
|
+
from selenium.webdriver.chrome.service import Service
|
|
7
|
+
from selenium.webdriver.common.by import By
|
|
8
|
+
from selenium.webdriver.common.keys import Keys
|
|
9
|
+
from selenium.webdriver.support.ui import WebDriverWait
|
|
10
|
+
from selenium.webdriver.support import expected_conditions as EC
|
|
11
|
+
from webdriver_manager.chrome import ChromeDriverManager
|
|
12
|
+
|
|
13
|
+
COLUMNS = ["Entity Name", "Contact Number", "Email Address", "Website URL", "Address", "Search Query", "Status"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_driver_service():
|
|
17
|
+
"""Linux/cloud: the system chromedriver (from packages.txt). Windows/macOS: ChromeDriverManager."""
|
|
18
|
+
system_path = "/usr/bin/chromedriver"
|
|
19
|
+
if os.path.exists(system_path):
|
|
20
|
+
return Service(system_path)
|
|
21
|
+
return Service(ChromeDriverManager().install())
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _noop(*args, **kwargs):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _read_place(driver, keyword):
|
|
29
|
+
nama_bisnis = "N/A"
|
|
30
|
+
try:
|
|
31
|
+
nama_bisnis = driver.find_element(By.TAG_NAME, "h1").text
|
|
32
|
+
except Exception:
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
no_telp = "-"
|
|
36
|
+
try:
|
|
37
|
+
btns = driver.find_elements(By.XPATH, '//button[contains(@data-item-id, "phone:tel:")]')
|
|
38
|
+
if btns:
|
|
39
|
+
txt = btns[0].get_attribute("aria-label") or ""
|
|
40
|
+
no_telp = txt.replace("Telepon: ", "").replace("Phone: ", "").strip()
|
|
41
|
+
else:
|
|
42
|
+
no_telp = "Not Available"
|
|
43
|
+
except Exception:
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
website = "-"
|
|
47
|
+
try:
|
|
48
|
+
web_btns = driver.find_elements(By.CSS_SELECTOR, 'a[data-item-id="authority"]')
|
|
49
|
+
if web_btns:
|
|
50
|
+
website = web_btns[0].get_attribute("href")
|
|
51
|
+
except Exception:
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
email_addr = "-"
|
|
55
|
+
try:
|
|
56
|
+
email_btns = driver.find_elements(By.XPATH, '//button[contains(@data-item-id, "email")]')
|
|
57
|
+
if email_btns:
|
|
58
|
+
txt = email_btns[0].get_attribute("aria-label") or ""
|
|
59
|
+
email_addr = txt.replace("Email: ", "").strip()
|
|
60
|
+
except Exception:
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
alamat = "-"
|
|
64
|
+
try:
|
|
65
|
+
addr_btns = driver.find_elements(By.XPATH, '//button[contains(@data-item-id, "address")]')
|
|
66
|
+
if addr_btns:
|
|
67
|
+
txt = addr_btns[0].get_attribute("aria-label") or ""
|
|
68
|
+
alamat = txt.replace("Alamat: ", "").replace("Address: ", "").strip()
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
"Entity Name": nama_bisnis,
|
|
74
|
+
"Contact Number": no_telp,
|
|
75
|
+
"Email Address": email_addr,
|
|
76
|
+
"Website URL": website,
|
|
77
|
+
"Address": alamat,
|
|
78
|
+
"Search Query": keyword,
|
|
79
|
+
"Status": "Pending Review",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def scrape(keyword, limit, headless=True, log=None, on_progress=None):
|
|
84
|
+
"""Search Google Maps for `keyword` and return up to `limit` rows with COLUMNS.
|
|
85
|
+
Optional callbacks: log(msg, type) and on_progress(phase, current, total), phase = harvest | extract | done."""
|
|
86
|
+
log = log or _noop
|
|
87
|
+
on_progress = on_progress or _noop
|
|
88
|
+
data = []
|
|
89
|
+
|
|
90
|
+
options = webdriver.ChromeOptions()
|
|
91
|
+
options.add_argument("--start-maximized")
|
|
92
|
+
options.add_argument("--log-level=3")
|
|
93
|
+
options.add_argument("--disable-blink-features=AutomationControlled")
|
|
94
|
+
# Required on Streamlit Cloud / Docker
|
|
95
|
+
options.add_argument("--no-sandbox")
|
|
96
|
+
options.add_argument("--disable-dev-shm-usage")
|
|
97
|
+
options.add_argument("--disable-gpu")
|
|
98
|
+
if headless:
|
|
99
|
+
options.add_argument("--headless=new")
|
|
100
|
+
|
|
101
|
+
driver = None
|
|
102
|
+
try:
|
|
103
|
+
log("Initializing WebDriver kernel...")
|
|
104
|
+
driver = webdriver.Chrome(service=get_driver_service(), options=options)
|
|
105
|
+
wait = WebDriverWait(driver, 20)
|
|
106
|
+
|
|
107
|
+
# 1. Navigation
|
|
108
|
+
driver.get("https://www.google.com/maps?hl=en")
|
|
109
|
+
time.sleep(3)
|
|
110
|
+
|
|
111
|
+
log(f"Acquiring target: **{keyword}**...")
|
|
112
|
+
try:
|
|
113
|
+
box = None
|
|
114
|
+
try:
|
|
115
|
+
box = driver.find_element(By.ID, "searchboxinput")
|
|
116
|
+
except Exception:
|
|
117
|
+
pass
|
|
118
|
+
if not box:
|
|
119
|
+
box = driver.find_element(By.NAME, "q")
|
|
120
|
+
|
|
121
|
+
if box:
|
|
122
|
+
box.clear()
|
|
123
|
+
box.send_keys(keyword)
|
|
124
|
+
time.sleep(1)
|
|
125
|
+
box.send_keys(Keys.ENTER)
|
|
126
|
+
else:
|
|
127
|
+
log("Search interface not detected.", "error")
|
|
128
|
+
return []
|
|
129
|
+
except Exception as e:
|
|
130
|
+
log(f"Search execution failed: {e}", "error")
|
|
131
|
+
return []
|
|
132
|
+
|
|
133
|
+
# 2. Collect place links (scrolling)
|
|
134
|
+
log("Harvesting endpoints (Scrolling)...")
|
|
135
|
+
time.sleep(3)
|
|
136
|
+
|
|
137
|
+
scroll_div = None
|
|
138
|
+
try:
|
|
139
|
+
scroll_div = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div[role="feed"]')))
|
|
140
|
+
except Exception:
|
|
141
|
+
time.sleep(3)
|
|
142
|
+
try:
|
|
143
|
+
scroll_div = driver.find_element(By.CSS_SELECTOR, 'div[role="feed"]')
|
|
144
|
+
except Exception:
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
if not scroll_div:
|
|
148
|
+
log("Feed container unavailable.", "error")
|
|
149
|
+
return []
|
|
150
|
+
|
|
151
|
+
count_found = 0
|
|
152
|
+
max_scroll_attempts = 5
|
|
153
|
+
no_new_data = 0
|
|
154
|
+
while count_found < limit:
|
|
155
|
+
driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scroll_div)
|
|
156
|
+
time.sleep(2.5)
|
|
157
|
+
current_count = len(driver.find_elements(By.CSS_SELECTOR, "a.hfpxzc"))
|
|
158
|
+
on_progress("harvest", current_count, limit)
|
|
159
|
+
if current_count == count_found:
|
|
160
|
+
no_new_data += 1
|
|
161
|
+
if no_new_data >= max_scroll_attempts:
|
|
162
|
+
log("End of list reached. Halting harvest.", "warning")
|
|
163
|
+
break
|
|
164
|
+
else:
|
|
165
|
+
no_new_data = 0
|
|
166
|
+
count_found = current_count
|
|
167
|
+
if count_found >= limit:
|
|
168
|
+
break
|
|
169
|
+
|
|
170
|
+
log("Indexing gathered URLs...", "info")
|
|
171
|
+
list_urls = []
|
|
172
|
+
for el in driver.find_elements(By.CSS_SELECTOR, "a.hfpxzc")[:limit]:
|
|
173
|
+
url = el.get_attribute("href")
|
|
174
|
+
if url:
|
|
175
|
+
list_urls.append(url)
|
|
176
|
+
log(f"Index secured: {len(list_urls)} endpoints ready. Commencing extraction...", "success")
|
|
177
|
+
|
|
178
|
+
# 3. Read each place
|
|
179
|
+
total = len(list_urls)
|
|
180
|
+
for i, url in enumerate(list_urls):
|
|
181
|
+
try:
|
|
182
|
+
on_progress("extract", i + 1, total)
|
|
183
|
+
driver.get(url)
|
|
184
|
+
try:
|
|
185
|
+
wait.until(EC.presence_of_element_located((By.TAG_NAME, "h1")))
|
|
186
|
+
except Exception:
|
|
187
|
+
time.sleep(2)
|
|
188
|
+
data.append(_read_place(driver, keyword))
|
|
189
|
+
except Exception as e:
|
|
190
|
+
log(f"Extraction failed for index {i+1}: {e}", "warning")
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
on_progress("done", total, total)
|
|
194
|
+
log("Sequence finished.", "success")
|
|
195
|
+
return data
|
|
196
|
+
|
|
197
|
+
except Exception as e:
|
|
198
|
+
log(f"Critical System Failure: {e}", "error")
|
|
199
|
+
return []
|
|
200
|
+
finally:
|
|
201
|
+
if driver:
|
|
202
|
+
driver.quit()
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""LeadScout from the command line.
|
|
2
|
+
|
|
3
|
+
Example:
|
|
4
|
+
python cli.py --roles "software house, IT consultant" --industries "technology" --location "Jakarta" --out prospects.csv
|
|
5
|
+
"""
|
|
6
|
+
import argparse
|
|
7
|
+
import csv
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from leadscout_core import COLUMNS, run_campaign
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def split_list(value):
|
|
15
|
+
return [v.strip() for v in value.split(",") if v.strip()]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main():
|
|
19
|
+
p = argparse.ArgumentParser(description="Research B2B prospects from public search results (Vulpfest Lab).")
|
|
20
|
+
p.add_argument("--roles", required=True, help='comma-separated roles or keywords, e.g. "software house, IT consultant"')
|
|
21
|
+
p.add_argument("--industries", required=True, help='comma-separated industry keywords, e.g. "technology"')
|
|
22
|
+
p.add_argument("--location", default="", help='optional location, e.g. "Jakarta"')
|
|
23
|
+
p.add_argument("--campaign", default="LeadScout", help="campaign name stored with each result")
|
|
24
|
+
p.add_argument("--source", choices=["web", "linkedin"], default="web", help="search the entire web (default) or LinkedIn profiles only")
|
|
25
|
+
p.add_argument("--max", type=int, default=10, help="maximum results per keyword (default 10, up to 50)")
|
|
26
|
+
p.add_argument("--out", metavar="FILE.csv", help="save the results as CSV")
|
|
27
|
+
a = p.parse_args()
|
|
28
|
+
|
|
29
|
+
roles, industries = split_list(a.roles), split_list(a.industries)
|
|
30
|
+
if not roles or not industries:
|
|
31
|
+
p.error("--roles and --industries cannot be empty")
|
|
32
|
+
print("Reminder: use public company information responsibly and follow data protection laws.", file=sys.stderr)
|
|
33
|
+
|
|
34
|
+
def log(msg):
|
|
35
|
+
print(re.sub(r"<[^>]+>", "", msg).strip(), file=sys.stderr)
|
|
36
|
+
|
|
37
|
+
rows = run_campaign(a.campaign, roles, industries, a.location.strip(), a.source, max(1, min(a.max, 50)), log_func=log)
|
|
38
|
+
if a.out:
|
|
39
|
+
with open(a.out, "w", newline="", encoding="utf-8-sig") as f:
|
|
40
|
+
w = csv.DictWriter(f, fieldnames=COLUMNS)
|
|
41
|
+
w.writeheader()
|
|
42
|
+
w.writerows(rows)
|
|
43
|
+
print(f"Saved {len(rows)} results to {a.out}", file=sys.stderr)
|
|
44
|
+
else:
|
|
45
|
+
for r in rows:
|
|
46
|
+
print(f"{r['Keyword']}\t{r['Page Title']}\t{r['URL']}")
|
|
47
|
+
print(f"\n{len(rows)} results", file=sys.stderr)
|
|
48
|
+
return 0 if rows else 1
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if __name__ == "__main__":
|
|
52
|
+
sys.exit(main())
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""LeadScout core: the search logic shared by the Streamlit app (app.py) and the command line (cli.py)."""
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
from ddgs import DDGS
|
|
6
|
+
except ImportError:
|
|
7
|
+
try:
|
|
8
|
+
from duckduckgo_search import DDGS
|
|
9
|
+
except ImportError:
|
|
10
|
+
DDGS = None
|
|
11
|
+
|
|
12
|
+
COLUMNS = ["Campaign", "Keyword", "Location", "Page Title", "URL"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def duckduckgo_search_api(query, max_results=15, target_type='linkedin', log_func=None):
|
|
16
|
+
results = []
|
|
17
|
+
try:
|
|
18
|
+
if DDGS is None:
|
|
19
|
+
raise ImportError("Missing library. Please run: pip install ddgs")
|
|
20
|
+
ddgs = DDGS()
|
|
21
|
+
search_results = ddgs.text(query, max_results=max_results)
|
|
22
|
+
|
|
23
|
+
if not search_results:
|
|
24
|
+
if log_func: log_func("<span style='color:#f87171;'>[!] DDG API returned 0 results (Backend Limit Hit).</span>")
|
|
25
|
+
return []
|
|
26
|
+
|
|
27
|
+
for r in search_results:
|
|
28
|
+
href = r.get('href', '')
|
|
29
|
+
title = r.get('title', '')
|
|
30
|
+
|
|
31
|
+
if target_type == 'linkedin':
|
|
32
|
+
# Strictly filter for LinkedIn Profiles only
|
|
33
|
+
if 'linkedin.com/in/' in href and '/dir/' not in href and '/jobs/' not in href:
|
|
34
|
+
clean_url = href.split('?')[0]
|
|
35
|
+
if clean_url not in [x['url'] for x in results]:
|
|
36
|
+
results.append({'title': title, 'url': clean_url})
|
|
37
|
+
else:
|
|
38
|
+
# Entire Web mode: accept all relevant URLs
|
|
39
|
+
if href not in [x['url'] for x in results]:
|
|
40
|
+
results.append({'title': title, 'url': href})
|
|
41
|
+
|
|
42
|
+
return results
|
|
43
|
+
except Exception as e:
|
|
44
|
+
if log_func: log_func(f"<span style='color:#f87171;'>[!] Internal DDG API error: {str(e)}</span>")
|
|
45
|
+
return []
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def build_query(role, industry, location="", target_type="web"):
|
|
49
|
+
location_dork = f' "{location}"' if location else ""
|
|
50
|
+
query = f'"{role}" {industry}{location_dork}'
|
|
51
|
+
return f"{query} site:linkedin.com/in" if target_type == "linkedin" else query
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run_campaign(campaign, roles, industries, location="", target_type="web", max_results=10, log_func=None, delay=1.5):
|
|
55
|
+
"""Same search flow as the web app; returns a list of rows with COLUMNS."""
|
|
56
|
+
rows, seen = [], set()
|
|
57
|
+
for role in roles:
|
|
58
|
+
if log_func: log_func(f"[*] Mining data for: {role} ...")
|
|
59
|
+
results = duckduckgo_search_api(build_query(role, industries[0], location, target_type), max_results, target_type, log_func)
|
|
60
|
+
time.sleep(delay) # respect API limits
|
|
61
|
+
count = 0
|
|
62
|
+
for item in results:
|
|
63
|
+
if count >= max_results:
|
|
64
|
+
break
|
|
65
|
+
if item["url"] in seen:
|
|
66
|
+
continue
|
|
67
|
+
seen.add(item["url"])
|
|
68
|
+
rows.append({"Campaign": campaign, "Keyword": role, "Location": location or "Global",
|
|
69
|
+
"Page Title": item["title"], "URL": item["url"]})
|
|
70
|
+
if log_func: log_func(f"[+] Found: {item['url']}")
|
|
71
|
+
count += 1
|
|
72
|
+
return rows
|