milenium 0.5.7__tar.gz → 1.0.2__tar.gz

{milenium-0.5.7 → milenium-1.0.2}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.4
 Name: milenium
-Version: 0.5.7
-Summary: OSINT toolkit
-Requires-Python: >=3.9
+Version: 1.0.2
+Summary: The most powerful OSINT aggregator
+Author: VexxDev200
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: requests
+Requires-Dist: aiohttp
 Requires-Dist: dnspython
 Requires-Dist: phonenumbers
 Requires-Dist: beautifulsoup4
@@ -12,6 +14,11 @@ Requires-Dist: rich
 Requires-Dist: click
 Requires-Dist: python-whois
 Requires-Dist: psycopg2-binary
+Requires-Dist: maigret
+Requires-Dist: sherlock-project
+Requires-Dist: holehe
+Requires-Dist: telethon
+Requires-Dist: PicImageSearch
 
 # milenium
 # exiftool

milenium-1.0.2/milenium/__init__.py

@@ -0,0 +1,53 @@
+import subprocess
+import sys
+import importlib
+
+
+def _ensure_deps():
+    """Автоустановка зависимостей при первом импорте."""
+    deps = {
+        "requests": "requests",
+        "aiohttp": "aiohttp",
+        "dns": "dnspython",
+        "phonenumbers": "phonenumbers",
+        "bs4": "beautifulsoup4",
+        "rich": "rich",
+        "click": "click",
+        "whois": "python-whois",
+        "psycopg2": "psycopg2-binary",
+        "telethon": "telethon",
+        "PicImageSearch": "PicImageSearch",
+    }
+def _ensure_deps():
+    deps = {
+            "requests": "requests",
+            "aiohttp": "aiohttp",
+            "dns": "dnspython",
+            "phonenumbers": "phonenumbers",
+            "bs4": "beautifulsoup4",
+            "rich": "rich",
+            "click": "click",
+            "whois": "python-whois",
+            "psycopg2": "psycopg2-binary",
+            "telethon": "telethon",
+            "PicImageSearch": "PicImageSearch",
+        }
+    for mod, pkg in deps.items():
+        try:
+            importlib.import_module(mod)
+        except ImportError:
+            try:
+                subprocess.check_call([sys.executable, "-m", "pip", "install", pkg, "--quiet"])
+            except Exception:
+                pass
+    # Прокидываем конфиг в окружение
+    try:
+        from milenium.modules import config
+        config.apply_to_env()
+    except Exception:
+        pass
+
+
+_ensure_deps()
+
+__version__ = "1.0.0"

milenium-1.0.2/milenium/cli.py

@@ -0,0 +1,180 @@
+import sys
+import json
+import time
+import asyncio
+import click
+from rich.console import Console
+from rich.table import Table
+from milenium.modules.logger import TeeLogger, progress
+from milenium.modules import (
+    neon_db, aggregator,
+    shodan_search, censys_search, virustotal_search,
+    abuseipdb_search, ipinfo_search, urlscan_search,
+    maigret_search, sherlock_search, holehe_search,
+    telegram_analysis, image_search,
+)
+
+console = Console()
+
+
+def _init_logger():
+    logger = TeeLogger("milenium.log", mode="a")
+    sys.stdout = logger
+    return logger
+
+
+def _save(command, target, module, data, target_type=None):
+    try:
+        neon_db.init()
+        neon_db.save(command, target, module, data, target_type=target_type)
+        progress("NeonDB: записано")
+    except Exception as e:
+        progress(f"NeonDB error: {e}")
+
+
+@click.group()
+def main():
+    """milenium — самый мощный OSINT агрегатор"""
+    pass
+
+
+@main.command()
+@click.argument("username")
+def user(username):
+    """Maigret + Sherlock + Holehe"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: user {username}")
+    res = aggregator.run(username=username)
+    _save("user", username, "username", res, "username")
+    console.print(json.dumps(res, indent=2, ensure_ascii=False, default=str))
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.argument("email")
+def email_cmd(email):
+    """Holehe + VirusTotal domain"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: email {email}")
+    res = aggregator.run(email=email)
+    _save("email", email, "email", res, "email")
+    console.print(json.dumps(res, indent=2, ensure_ascii=False, default=str))
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.argument("ip")
+def ip_cmd(ip):
+    """Shodan + Censys + VirusTotal + AbuseIPDB + ipinfo"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: ip {ip}")
+    res = aggregator.run(ip=ip)
+    _save("ip", ip, "ip", res, "ip")
+    console.print(json.dumps(res, indent=2, ensure_ascii=False, default=str))
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.argument("url")
+def url_cmd(url):
+    """urlscan.io анализ"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: url {url}")
+    res = aggregator.run(url=url)
+    _save("url", url, "url", res, "url")
+    console.print(json.dumps(res, indent=2, ensure_ascii=False, default=str))
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.argument("username")
+def tg(username):
+    """Telethon: инфа по TG-аккаунту"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: telegram {username}")
+    res = asyncio.run(telegram_analysis.check_user(username))
+    _save("telegram", username, "telegram", res, "username")
+    for k, v in res.items():
+        console.print(f"[bold]{k}:[/bold] {v}")
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.argument("image_path")
+@click.option("--engine", default="yandex")
+def img(image_path, engine):
+    """PicImageSearch: обратный поиск по фото"""
+    logger = _init_logger()
+    t0 = time.time()
+    progress(f"старт: image search {engine}")
+    res = asyncio.run(image_search.search(image_path, engine))
+    _save("image", image_path, "image", res, "file")
+    console.print(json.dumps(res, indent=2, ensure_ascii=False, default=str))
+    progress(f"готово за {time.time() - t0:.1f}s")
+    logger.close()
+
+
+@main.command()
+@click.option("--key", default=None, help="Какой ключ задать")
+@click.option("--value", default=None, help="Значение")
+def config_cmd(key, value):
+    """Просмотр и настройка конфига"""
+    from milenium.modules import config
+    if key and value:
+        config.set_key(key, value)
+        console.print(f"[green]Сохранено:[/green] {key}")
+        return
+    cfg = config.all_keys()
+    console.print(f"[bold]Конфиг:[/bold] {config.config_path()}")
+    for k, v in cfg.items():
+        shown = v if not v else (v[:8] + "..." if len(v) > 12 else v)
+        console.print(f"  [red]{k}[/red] = {shown or '[empty]'}")
+
+
+@main.command()
+@click.option("--target", default=None)
+@click.option("--stats", is_flag=True)
+def db_neon(target, stats):
+    """NeonDB статистика и поиск"""
+    logger = _init_logger()
+    try:
+        neon_db.init()
+        if stats:
+            s = neon_db.stats()
+            console.print(f"[bold]Всего:[/bold] {s['total']}")
+            console.print(f"[bold]Findings:[/bold] {s['findings']}")
+            console.print(f"[bold]Leaks:[/bold] {s['leaks']}")
+        else:
+            rows = neon_db.query(target=target, limit=50)
+            table = Table(title="NeonDB")
+            table.add_column("Время")
+            table.add_column("Команда")
+            table.add_column("Цель")
+            table.add_column("Модуль")
+            for r in rows:
+                table.add_row(str(r["ts"])[:19], r["command"], r["target"], r["module"])
+            console.print(table)
+    except Exception as e:
+        console.print(f"[bold red]Ошибка Neon:[/bold red] {e}")
+    logger.close()
+
+
+@main.command()
+def tui():
+    """TUI интерфейс"""
+    from milenium.tui import interactive
+    interactive()
+
+
+if __name__ == "__main__":
+    main()

milenium-1.0.2/milenium/modules/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    "logger", "neon_db", "aggregator", "dorker",
+    "username", "social", "email", "phone", "ip", "domain", "breach", "report",
+    "pwned_passwords", "breach_local", "hibp", "dehashed", "leakcheck",
+    "shodan_lite", "shodan_search", "censys_search", "virustotal_search",
+    "abuseipdb_search", "ipinfo_search", "urlscan_search",
+    "telegram_osint", "telegram_analysis", "image_search",
+    "maigret_search", "sherlock_search", "holehe_search",
+]

milenium-1.0.2/milenium/modules/abuseipdb_search.py

@@ -0,0 +1,30 @@
+import os
+import aiohttp
+
+from milenium.modules import config
+ABUSE_KEY = config.get("ABUSEIPDB_KEY", "")
+
+
+async def check(ip: str) -> dict:
+    """AbuseIPDB: жалобы на IP."""
+    if not ABUSE_KEY:
+        return {"error": "ABUSEIPDB_KEY not set"}
+    url = "https://api.abuseipdb.com/api/v2/check"
+    params = {"ipAddress": ip, "maxAgeInDays": 90}
+    headers = {"Key": ABUSE_KEY, "Accept": "application/json"}
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, params=params, headers=headers, timeout=15) as resp:
+                if resp.status != 200:
+                    return {"error": f"HTTP {resp.status}"}
+                data = await resp.json()
+                d = data.get("data", {})
+                return {
+                    "abuse_score": d.get("abuseConfidenceScore"),
+                    "total_reports": d.get("totalReports"),
+                    "country": d.get("countryCode"),
+                    "isp": d.get("isp"),
+                    "domain": d.get("domain"),
+                }
+    except Exception as e:
+        return {"error": str(e)}

milenium-1.0.2/milenium/modules/aggregator.py

@@ -0,0 +1,43 @@
+import asyncio
+from milenium.modules import (
+    shodan_search, censys_search, virustotal_search,
+    abuseipdb_search, ipinfo_search, urlscan_search,
+    maigret_search, sherlock_search, holehe_search,
+)
+
+
+async def run_async(email=None, username=None, phone=None, ip=None, url=None):
+    tasks = {}
+
+    if ip:
+        tasks["shodan"] = shodan_search.check_ip(ip)
+        tasks["censys"] = censys_search.check_host(ip)
+        tasks["virustotal"] = virustotal_search.check_ip(ip)
+        tasks["abuseipdb"] = abuseipdb_search.check(ip)
+        tasks["ipinfo"] = ipinfo_search.check(ip)
+
+    if username:
+        tasks["maigret"] = maigret_search.check(username)
+        tasks["sherlock"] = sherlock_search.check(username)
+
+    if email:
+        tasks["holehe"] = holehe_search.check(email)
+        tasks["virustotal_domain"] = virustotal_search.check_domain(email.split("@")[-1])
+
+    if url:
+        tasks["urlscan"] = urlscan_search.check(url)
+
+    results = {}
+    if tasks:
+        keys = list(tasks.keys())
+        values = await asyncio.gather(*tasks.values(), return_exceptions=True)
+        for k, v in zip(keys, values):
+            if isinstance(v, Exception):
+                results[k] = {"error": str(v)}
+            else:
+                results[k] = v
+    return results
+
+
+def run(**kwargs):
+    return asyncio.run(run_async(**kwargs))

milenium-1.0.2/milenium/modules/censys_search.py

@@ -0,0 +1,30 @@
+import os
+import aiohttp
+
+CENSYS_TOKEN = os.environ.get("CENSYS_TOKEN", "censys_UJptWhGQ_L57vdqJiF1FQEd1VtTVArqsX")
+
+
+async def check_host(ip: str) -> dict:
+    """Censys API: хосты, сервисы, TLS."""
+    if not CENSYS_TOKEN:
+        return {"error": "CENSYS_TOKEN not set"}
+    url = f"https://search.censys.io/api/v2/hosts/{ip}"
+    headers = {"Authorization": f"Bearer {CENSYS_TOKEN}"}
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers, timeout=15) as resp:
+                if resp.status == 404:
+                    return {"found": False}
+                if resp.status != 200:
+                    return {"error": f"HTTP {resp.status}"}
+                data = await resp.json()
+                result = data.get("result", {})
+                services = result.get("services", [])
+                return {
+                    "services": [s.get("service_name") for s in services],
+                    "ports": [s.get("port") for s in services],
+                    "os": result.get("operating_system", {}).get("product"),
+                    "country": result.get("location", {}).get("country"),
+                }
+    except Exception as e:
+        return {"error": str(e)}

milenium-1.0.2/milenium/modules/holehe_search.py

@@ -0,0 +1,17 @@
+import asyncio
+import json
+
+
+async def check(email: str) -> dict:
+    """Holehe: проверка email по 120+ платформам."""
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "holehe", email, "--only-used", "--no-color",
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+        )
+        stdout, _ = await proc.communicate()
+        lines = stdout.decode("utf-8", errors="ignore").splitlines()
+        found = [l for l in lines if "[+]" in l]
+        return {"found": len(found), "sites": found[:100]}
+    except Exception as e:
+        return {"error": str(e)}

milenium-1.0.2/milenium/modules/image_search.py

@@ -0,0 +1,24 @@
+import asyncio
+from PicImageSearch import Network, Yandex, Google, Bing, Tineye
+
+
+async def search(image_path: str, engine: str = "yandex") -> dict:
+    """PicImageSearch: обратный поиск по фото."""
+    engines = {"yandex": Yandex, "google": Google, "bing": Bing, "tineye": Tineye}
+    cls = engines.get(engine.lower())
+    if not cls:
+        return {"error": f"unknown engine: {engine}"}
+    try:
+        async with Network() as client:
+            searcher = cls(client=client)
+            resp = await searcher.search(file=image_path)
+            results = []
+            for item in resp.raw[:10]:
+                results.append({
+                    "title": getattr(item, "title", None),
+                    "url": getattr(item, "url", None),
+                    "thumbnail": getattr(item, "thumbnail", None),
+                })
+            return {"engine": engine, "results": results}
+    except Exception as e:
+        return {"error": str(e)}

milenium-1.0.2/milenium/modules/ipinfo_search.py

@@ -0,0 +1,28 @@
+import os
+import aiohttp
+
+from milenium.modules import config
+IPINFO_TOKEN = config.get("IPINFO_TOKEN", "")
+
+
+async def check(ip: str) -> dict:
+    """ipinfo.io: гео, ASN, компания."""
+    if not IPINFO_TOKEN:
+        return {"error": "IPINFO_TOKEN not set"}
+    url = f"https://ipinfo.io/{ip}/json?token={IPINFO_TOKEN}"
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, timeout=15) as resp:
+                if resp.status != 200:
+                    return {"error": f"HTTP {resp.status}"}
+                data = await resp.json()
+                return {
+                    "city": data.get("city"),
+                    "region": data.get("region"),
+                    "country": data.get("country"),
+                    "org": data.get("org"),
+                    "timezone": data.get("timezone"),
+                    "loc": data.get("loc"),
+                }
+    except Exception as e:
+        return {"error": str(e)}

{milenium-0.5.7 → milenium-1.0.2}/milenium/modules/logger.py

@@ -32,7 +32,7 @@ class TeeLogger:
 
     def _write_header(self):
         ts = datetime.now().isoformat()
-        self.file.write(f"\n{'='*60}\n[SESSION START] {ts}\n{'='*60}\n")
+        self.file.write(f"\n{'=' * 60}\n[SESSION START] {ts}\n{'=' * 60}\n")
         self.file.flush()
 
     def write(self, message):

milenium-1.0.2/milenium/modules/maigret_search.py

@@ -0,0 +1,17 @@
+import asyncio
+import json
+
+
+async def check(username: str) -> dict:
+    """Maigret: поиск по 3000+ сайтам."""
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "maigret", username, "--json", "simple", "--no-progressbar",
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+        )
+        stdout, _ = await proc.communicate()
+        data = json.loads(stdout.decode("utf-8", errors="ignore"))
+        found = [k for k, v in data.items() if v.get("status") == "claimed"]
+        return {"found": len(found), "sites": found[:100]}
+    except Exception as e:
+        return {"error": str(e)}

{milenium-0.5.7 → milenium-1.0.2}/milenium/modules/neon_db.py

@@ -2,21 +2,28 @@ import os
 import json
 import psycopg2
 import psycopg2.extras
+from milenium.modules import config
 
-DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://neondb_owner:npg_JNAjnTaz0UK5@ep-twilight-bar-b1ofdyot-pooler.c-5.eu-central-1.aws.neon.tech/neondb?sslmode=require&channel_binding=require")
+DATABASE_URL = ""
 
 
-def _conn():
+def _get_url():
+    global DATABASE_URL
     if not DATABASE_URL:
+        DATABASE_URL = config.get("DATABASE_URL", "")
+    return DATABASE_URL
+
+
+def _conn():
+    url = _get_url()
+    if not url:
         raise RuntimeError("DATABASE_URL not set")
-    return psycopg2.connect(DATABASE_URL, sslmode="require")
+    return psycopg2.connect(url, sslmode="require")
 
 
 def init():
-    """Создаёт расширенные таблицы."""
     with _conn() as conn:
         with conn.cursor() as cur:
-            # Основная таблица результатов
             cur.execute("""
                 CREATE TABLE IF NOT EXISTS results (
                     id SERIAL PRIMARY KEY,
@@ -29,7 +36,6 @@ def init():
                     data JSONB
                 )
             """)
-            # Отдельная таблица для найденных URL/ссылок
             cur.execute("""
                 CREATE TABLE IF NOT EXISTS findings (
                     id SERIAL PRIMARY KEY,
@@ -41,7 +47,6 @@ def init():
                     meta JSONB
                 )
             """)
-            # Таблица для утечек
             cur.execute("""
                 CREATE TABLE IF NOT EXISTS leaks (
                     id SERIAL PRIMARY KEY,
@@ -53,7 +58,6 @@ def init():
                     breach TEXT
                 )
             """)
-            # Таблица сессий
             cur.execute("""
                 CREATE TABLE IF NOT EXISTS sessions (
                     id SERIAL PRIMARY KEY,
@@ -66,7 +70,6 @@ def init():
 
 
 def save(command, target, module, data, target_type=None):
-    """Сохраняет результат + разбирает его на findings/leaks."""
     found_count = 0
     if isinstance(data, dict):
         for k, v in data.items():
@@ -79,10 +82,9 @@ def save(command, target, module, data, target_type=None):
                 """INSERT INTO results (command, target, target_type, module, found_count, data)
                    VALUES (%s, %s, %s, %s, %s, %s)""",
                 (command, target, target_type, module, found_count,
-                 json.dumps(data, ensure_ascii=False))
+                 json.dumps(data, ensure_ascii=False, default=str))
             )
 
-            # Разбираем findings
             if isinstance(data, dict):
                 for key, val in data.items():
                     if isinstance(val, dict) and "url" in val:
@@ -90,7 +92,7 @@ def save(command, target, module, data, target_type=None):
                             """INSERT INTO findings (target, source, url, found, meta)
                                VALUES (%s, %s, %s, %s, %s)""",
                             (target, key, val.get("url"), val.get("found", False),
-                             json.dumps(val, ensure_ascii=False))
+                             json.dumps(val, ensure_ascii=False, default=str))
                         )
                     elif isinstance(val, list):
                         for item in val:
@@ -99,10 +101,15 @@ def save(command, target, module, data, target_type=None):
                                     """INSERT INTO findings (target, source, url, found, meta)
                                        VALUES (%s, %s, %s, %s, %s)""",
                                     (target, key, item.get("url"), True,
-                                     json.dumps(item, ensure_ascii=False))
+                                     json.dumps(item, ensure_ascii=False, default=str))
+                                )
+                            elif isinstance(item, str) and item.startswith("http"):
+                                cur.execute(
+                                    """INSERT INTO findings (target, source, url, found, meta)
+                                       VALUES (%s, %s, %s, %s, %s)""",
+                                    (target, key, item, True, json.dumps({"url": item}))
                                 )
 
-            # Разбираем утечки
             if isinstance(data, dict) and "breaches" in data:
                 for b in data.get("breaches", []):
                     cur.execute(
@@ -163,19 +170,22 @@ def leaks(target=None, limit=100):
 
 
 def stats():
-    with _conn() as conn:
-        with conn.cursor() as cur:
-            cur.execute("SELECT COUNT(*) FROM results")
-            total = cur.fetchone()[0]
-            cur.execute("SELECT COUNT(*) FROM findings WHERE found = TRUE")
-            findings_count = cur.fetchone()[0]
-            cur.execute("SELECT COUNT(*) FROM leaks")
-            leaks_count = cur.fetchone()[0]
-            cur.execute("SELECT module, COUNT(*) FROM results GROUP BY module")
-            by_module = dict(cur.fetchall())
-    return {
-        "total": total,
-        "findings": findings_count,
-        "leaks": leaks_count,
-        "by_module": by_module,
-    }
+    try:
+        with _conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT COUNT(*) FROM results")
+                total = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM findings WHERE found = TRUE")
+                findings_count = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM leaks")
+                leaks_count = cur.fetchone()[0]
+                cur.execute("SELECT module, COUNT(*) FROM results GROUP BY module")
+                by_module = dict(cur.fetchall())
+        return {
+            "total": total,
+            "findings": findings_count,
+            "leaks": leaks_count,
+            "by_module": by_module,
+        }
+    except Exception as e:
+        return {"total": 0, "findings": 0, "leaks": 0, "by_module": {}, "error": str(e)}

milenium-1.0.2/milenium/modules/sherlock_search.py

@@ -0,0 +1,16 @@
+import asyncio
+import json
+
+
+async def check(username: str) -> dict:
+    """Sherlock: поиск по 3000+ сетям."""
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "sherlock", username, "--json",
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+        )
+        stdout, _ = await proc.communicate()
+        data = json.loads(stdout.decode("utf-8", errors="ignore"))
+        return {"found": len(data), "sites": list(data.keys())[:100]}
+    except Exception as e:
+        return {"error": str(e)}

milenium-1.0.2/milenium/modules/shodan_search.py

@@ -0,0 +1,31 @@
+import os
+import aiohttp
+
+from milenium.modules import config
+SHODAN_KEY = config.get("SHODAN_API_KEY", "")
+
+
+async def check_ip(ip: str) -> dict:
+    """Shodan API: порты, баннеры, CVE, гео."""
+    if not SHODAN_KEY:
+        return {"error": "SHODAN_API_KEY not set"}
+    url = f"https://api.shodan.io/shodan/host/{ip}?key={SHODAN_KEY}&minify=true"
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, timeout=15) as resp:
+                if resp.status == 404:
+                    return {"found": False}
+                if resp.status != 200:
+                    return {"error": f"HTTP {resp.status}"}
+                data = await resp.json()
+                return {
+                    "org": data.get("org"),
+                    "os": data.get("os"),
+                    "ports": data.get("ports", []),
+                    "hostnames": data.get("hostnames", []),
+                    "vulns": list(data.get("vulns", {}).keys()),
+                    "country": data.get("country_name"),
+                    "city": data.get("city"),
+                }
+    except Exception as e:
+        return {"error": str(e)}