moriarty-project 0.1.6__py3-none-any.whl

moriarty/cli/user.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+import re
+
+import typer
+import httpx
+from rich import box
+from rich.table import Table
+
+from .async_utils import run_async
+from .output import console, emit_json, emit_yaml, ensure_state
+from ..dsl.loader import TemplateLoader
+from ..modules.template_executor import ExecutionResult, TemplateExecutor
+
+app = typer.Typer(add_completion=False, help="Enumeração de usernames.")
+
+
+@app.command("enum")
+def enum(
+    ctx: typer.Context,
+    username: str = typer.Option(..., "--username", "-u", help="Username a pesquisar."),
+    sites: str = typer.Option("top20", help="Sites: top20, all, tag:X, ou caminho para .yml"),
+    json_output: Optional[str] = typer.Option(None, "--json", help="Escreve saída JSON no caminho ou '-'"),
+    retry: int = typer.Option(0, min=0, help="Tentativas de retry por site."),
+    nsfw: bool = typer.Option(False, "--nsfw", help="Include checking of NSFW sites from default list."),
+    print_all: bool = typer.Option(False, "--print-all", help="Output sites where the username was not found."),
+    print_found: bool = typer.Option(False, "--print-found", help="Output sites where the username was found."),
+    timeout_override: Optional[float] = typer.Option(None, "--timeout", help="Time (in seconds) to wait for response to requests."),
+) -> None:
+    """Enumera username em múltiplos sites."""
+    import logging
+    
+    state = ensure_state(ctx)
+    
+    # Override timeout se especificado
+    if timeout_override is not None:
+        state.options.timeout = timeout_override
+    
+    # Suprime logs INFO/WARNING de bibliotecas externas
+    if not state.options.verbose:
+        logging.getLogger("httpx").setLevel(logging.ERROR)
+        logging.getLogger("httpcore").setLevel(logging.ERROR)
+        logging.getLogger("moriarty.modules.template_executor").setLevel(logging.ERROR)
+    
+    # Carrega templates
+    loader = TemplateLoader()
+    loader.load_builtin()
+    
+    # Filtra templates
+    if sites == "all":
+        templates = loader.list_templates()
+    elif sites == "top20":
+        templates = loader.list_templates(tag="top20")
+    elif sites.startswith("tag:"):
+        tag = sites[4:]
+        templates = loader.list_templates(tag=tag)
+    else:
+        # Caminho para arquivo/diretório
+        from pathlib import Path
+        path = Path(sites)
+        if path.is_file():
+            templates = [loader.load_from_path(path)]
+        elif path.is_dir():
+            templates = loader.load_from_directory(path)
+        else:
+            raise typer.BadParameter(f"Invalid sites specification: {sites}")
+    
+    # Filtra templates
+    # 1. Filtra habilitados
+    templates = [t for t in templates if t.enabled]
+    
+    # 2. Filtra headless se necessário
+    if state.options.headless == "never":
+        templates = [t for t in templates if not t.require_headless]
+    
+    # 3. Filtra NSFW se não solicitado
+    if not nsfw:
+        templates = [t for t in templates if not t.nsfw]
+    
+    if not templates:
+        console.print("[warning]Nenhum template encontrado para executar.")
+        return
+    
+    allowed_vars = {"username"}
+    templates = _filter_templates_by_variables(templates, allowed_vars)
+
+    console.print(f"[info]🔎 Buscando username em {len(templates)} sites...")
+    console.print()
+    
+    # Executa com spinner
+    with console.status("[cyan]Verificando perfis...", spinner="dots"):
+        cookie_store: Dict[str, httpx.Cookies] = {}
+        executor = TemplateExecutor(timeout=state.options.timeout, cookie_store=cookie_store)
+        results = run_async(_execute_all(executor, templates, username, state.options.concurrency))
+    
+    # Saída
+    output_format = ("json" if json_output else state.options.format).lower()
+    
+    # Filtra resultados para exibição
+    if print_found:
+        # Mostra apenas encontrados
+        filtered_results = [r for r in results if r.exists]
+    elif print_all:
+        # Mostra todos (padrão se nenhum filtro)
+        filtered_results = results
+    else:
+        # Mostra todos por padrão
+        filtered_results = results
+    
+    if output_format == "json":
+        payload = _results_to_dict(filtered_results, username)
+        emit_json(payload, json_output, state.options.output)
+    elif output_format == "yaml":
+        payload = _results_to_dict(filtered_results, username)
+        emit_yaml(payload, json_output, state.options.output)
+    else:
+        _render_table(filtered_results, username, show_not_found=print_all or not print_found)
+
+
+async def _execute_all(
+    executor: TemplateExecutor,
+    templates: List[Any],
+    username: str,
+    concurrency: int,
+) -> List[ExecutionResult]:
+    """Executa todos os templates concorrentemente."""
+    semaphore = asyncio.Semaphore(concurrency)
+    
+    async def bounded_execute(template: Any) -> ExecutionResult:
+        async with semaphore:
+            return await executor.execute(template, {"username": username})
+    
+    tasks = [bounded_execute(t) for t in templates]
+    return await asyncio.gather(*tasks)
+
+
+def _render_table(results: List[ExecutionResult], username: str, show_not_found: bool = True) -> None:
+    """Renderiza resultados em tabela."""
+    table = Table(title=f"Username Enumeration · {username}", box=box.SIMPLE_HEAVY)
+    table.add_column("Site", style="cyan", no_wrap=True)
+    table.add_column("Exists", style="white", no_wrap=True)
+    table.add_column("Confidence", style="white", no_wrap=True)
+    table.add_column("Extracted", style="dim")
+    table.add_column("URL", style="blue", overflow="fold")
+    
+    # Ordena por exists e confidence
+    sorted_results = sorted(results, key=lambda r: (r.exists, r.confidence), reverse=True)
+    
+    for result in sorted_results:
+        # Pula não encontrados se show_not_found=False
+        if not show_not_found and not result.exists:
+            continue
+        
+        exists_icon = "✓" if result.exists else "✗"
+        exists_color = "green" if result.exists else "dim"
+        
+        confidence_str = f"{result.confidence:.2f}"
+        
+        extracted_str = ", ".join(f"{k}={v}" for k, v in result.extracted.items()) if result.extracted else "—"
+        
+        table.add_row(
+            result.site,
+            f"[{exists_color}]{exists_icon}[/{exists_color}]",
+            confidence_str,
+            extracted_str,
+            result.url,
+        )
+    
+    console.print(table)
+    
+    # Estatísticas
+    total = len(results)
+    found = sum(1 for r in results if r.exists)
+    console.print(f"\n[info]Total: {total} sites · Found: {found} ({found/total*100:.1f}%)")
+
+
+def _results_to_dict(results: List[ExecutionResult], username: str) -> Dict[str, Any]:
+    """Converte resultados para dict."""
+    return {
+        "username": username,
+        "total_sites": len(results),
+        "found_count": sum(1 for r in results if r.exists),
+        "results": [
+            {
+                "site": r.site,
+                "url": r.url,
+                "exists": r.exists,
+                "confidence": r.confidence,
+                "extracted": r.extracted,
+                "page_hash": r.page_hash,
+                "latency_ms": r.latency_ms,
+                "status_code": r.status_code,
+                "error": r.error,
+            }
+            for r in results
+        ],
+    }
+
+
+_PLACEHOLDER_PATTERN = re.compile(r"\{([a-zA-Z0-9_]+)\}")
+
+
+def _filter_templates_by_variables(templates: List[Any], allowed: set[str]) -> List[Any]:
+    filtered: List[Any] = []
+    for template in templates:
+        placeholders = _collect_placeholders(template)
+        if placeholders - allowed:
+            continue
+        filtered.append(template)
+    return filtered
+
+
+def _collect_placeholders(template: Any) -> set[str]:
+    values = [template.url_template]
+    values.extend(template.headers.values())
+    if template.body:
+        values.append(str(template.body))
+    placeholders: set[str] = set()
+    for value in values:
+        for match in _PLACEHOLDER_PATTERN.findall(str(value)):
+            placeholders.add(match)
+    return placeholders
+
+
+__all__ = ["app"]

moriarty/core/cache_backend.py

@@ -0,0 +1,223 @@
+"""Cache backends with LRU eviction and persistence support."""
+from __future__ import annotations
+
+import pickle
+import sqlite3
+import time
+from collections import OrderedDict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, Optional, Tuple
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class CacheRecord:
+    """Representa um registro em cache."""
+
+    key: Tuple[str, str]
+    value: Any
+    expires_at: float
+    ttl: int
+    last_access: float
+
+
+class BaseCacheBackend:
+    """Interface base para backends de cache."""
+
+    name: str = "base"
+
+    def __init__(self):
+        self.metrics = {
+            "evictions": 0,
+            "writes": 0,
+            "reads": 0,
+        }
+
+    def get(self, key: Tuple[str, str]) -> Optional[CacheRecord]:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def set(self, key: Tuple[str, str], value: Any, ttl: int) -> None:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def delete(self, key: Tuple[str, str]) -> None:
+        raise NotImplementedError
+
+    def clear(self) -> None:
+        raise NotImplementedError
+
+    def purge_expired(self) -> None:
+        raise NotImplementedError
+
+    def warm(self, records: Iterable[CacheRecord]) -> None:
+        for record in records:
+            now = time.time()
+            remaining_ttl = int(record.expires_at - now)
+            if remaining_ttl > 0:
+                self.set(record.key, record.value, remaining_ttl)
+
+    def stats(self) -> Dict[str, Any]:
+        return {"backend": self.name, **self.metrics}
+
+
+class MemoryCacheBackend(BaseCacheBackend):
+    """Cache em memória com política LRU."""
+
+    name = "memory"
+
+    def __init__(self, max_size: int, eviction_policy: str = "lru"):
+        super().__init__()
+        self.max_size = max_size
+        self.eviction_policy = eviction_policy.lower()
+        self._store: OrderedDict[Tuple[str, str], CacheRecord] = OrderedDict()
+
+    def get(self, key: Tuple[str, str]) -> Optional[CacheRecord]:
+        record = self._store.get(key)
+        if not record:
+            return None
+
+        if record.expires_at <= time.time():
+            self._store.pop(key, None)
+            return None
+
+        if self.eviction_policy == "lru":
+            self._store.move_to_end(key)
+
+        record.last_access = time.time()
+        self.metrics["reads"] += 1
+        return record
+
+    def set(self, key: Tuple[str, str], value: Any, ttl: int) -> None:
+        expires_at = time.time() + ttl
+        record = CacheRecord(key=key, value=value, expires_at=expires_at, ttl=ttl, last_access=time.time())
+        self._store[key] = record
+        if self.eviction_policy == "lru":
+            self._store.move_to_end(key)
+        self.metrics["writes"] += 1
+        self._evict_if_needed()
+
+    def delete(self, key: Tuple[str, str]) -> None:
+        self._store.pop(key, None)
+
+    def clear(self) -> None:
+        self._store.clear()
+
+    def purge_expired(self) -> None:
+        now = time.time()
+        expired_keys = [key for key, record in self._store.items() if record.expires_at <= now]
+        for key in expired_keys:
+            self._store.pop(key, None)
+
+    def _evict_if_needed(self) -> None:
+        while len(self._store) > self.max_size > 0:
+            key, _ = self._store.popitem(last=False)
+            self.metrics["evictions"] += 1
+
+
+class SQLiteCacheBackend(BaseCacheBackend):
+    """Cache persistente em SQLite com política LRU."""
+
+    name = "sqlite"
+
+    def __init__(self, db_path: Path, max_size: int):
+        super().__init__()
+        self.db_path = db_path
+        self.max_size = max_size
+        self._connection = sqlite3.connect(str(db_path), check_same_thread=False)
+        self._connection.execute(
+            """
+            CREATE TABLE IF NOT EXISTS cache (
+                cache_key TEXT PRIMARY KEY,
+                value BLOB NOT NULL,
+                expires_at REAL NOT NULL,
+                last_access REAL NOT NULL,
+                ttl INTEGER NOT NULL
+            )
+            """
+        )
+        self._connection.commit()
+
+    def get(self, key: Tuple[str, str]) -> Optional[CacheRecord]:
+        cursor = self._connection.execute(
+            "SELECT value, expires_at, last_access, ttl FROM cache WHERE cache_key = ?",
+            (self._serialize_key(key),),
+        )
+        row = cursor.fetchone()
+        if not row:
+            return None
+
+        value_blob, expires_at, _, ttl = row
+        if expires_at <= time.time():
+            self.delete(key)
+            return None
+
+        value = pickle.loads(value_blob)
+        record = CacheRecord(
+            key=key,
+            value=value,
+            expires_at=expires_at,
+            ttl=ttl,
+            last_access=time.time(),
+        )
+        self._connection.execute(
+            "UPDATE cache SET last_access = ? WHERE cache_key = ?",
+            (record.last_access, self._serialize_key(key)),
+        )
+        self._connection.commit()
+        self.metrics["reads"] += 1
+        return record
+
+    def set(self, key: Tuple[str, str], value: Any, ttl: int) -> None:
+        expires_at = time.time() + ttl
+        blob = pickle.dumps(value)
+        last_access = time.time()
+        self._connection.execute(
+            "REPLACE INTO cache (cache_key, value, expires_at, last_access, ttl) VALUES (?, ?, ?, ?, ?)",
+            (self._serialize_key(key), blob, expires_at, last_access, ttl),
+        )
+        self._connection.commit()
+        self.metrics["writes"] += 1
+        self._evict_if_needed()
+
+    def delete(self, key: Tuple[str, str]) -> None:
+        self._connection.execute(
+            "DELETE FROM cache WHERE cache_key = ?",
+            (self._serialize_key(key),),
+        )
+        self._connection.commit()
+
+    def clear(self) -> None:
+        self._connection.execute("DELETE FROM cache")
+        self._connection.commit()
+
+    def purge_expired(self) -> None:
+        now = time.time()
+        self._connection.execute("DELETE FROM cache WHERE expires_at <= ?", (now,))
+        self._connection.commit()
+
+    def warm(self, records: Iterable[CacheRecord]) -> None:
+        for record in records:
+            remaining_ttl = int(record.expires_at - time.time())
+            if remaining_ttl > 0:
+                self.set(record.key, record.value, remaining_ttl)
+
+    def _serialize_key(self, key: Tuple[str, str]) -> str:
+        return "|".join(key)
+
+    def _evict_if_needed(self) -> None:
+        if self.max_size <= 0:
+            return
+        cursor = self._connection.execute("SELECT COUNT(*) FROM cache")
+        count = cursor.fetchone()[0]
+        if count <= self.max_size:
+            return
+        overflow = count - self.max_size
+        self._connection.execute(
+            "DELETE FROM cache WHERE cache_key IN (SELECT cache_key FROM cache ORDER BY last_access ASC LIMIT ?)",
+            (overflow,),
+        )
+        self._connection.commit()
+        self.metrics["evictions"] += overflow