moriarty-project 0.1.6__py3-none-any.whl

moriarty/modules/wayback_discovery.py

@@ -0,0 +1,234 @@
+"""Descoberta de URLs históricas via Wayback Machine."""
+import asyncio
+from dataclasses import dataclass
+from typing import List, Optional, Set
+from urllib.parse import urlparse
+
+import httpx
+import structlog
+from rich.progress import Progress
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class WaybackURL:
+    """URL histórica do Wayback Machine."""
+    url: str
+    timestamp: str
+    status_code: int
+    is_active: bool = False
+
+
+class WaybackDiscovery:
+    """Descoberta de URLs históricas via Wayback Machine."""
+    
+    def __init__(
+        self,
+        domain: str,
+        validate: bool = True,
+        filter_extensions: Optional[List[str]] = None,
+        filter_status_codes: Optional[List[int]] = None,
+        years_back: int = 5,
+        timeout: float = 10.0,
+        extract_params: bool = True,
+        detect_admin: bool = True,
+    ):
+        self.domain = domain.lower().strip()
+        self.validate = validate
+        self.filter_extensions = filter_extensions
+        self.filter_status_codes = filter_status_codes or [200]
+        self.years_back = years_back
+        self.timeout = timeout
+        self.extract_params = extract_params
+        self.detect_admin = detect_admin
+        self.urls: Set[str] = set()
+        self.parameters: Set[str] = set()
+        self.admin_urls: Set[str] = set()
+    
+    async def discover(self) -> List[WaybackURL]:
+        """Descobre URLs históricas."""
+        logger.info("wayback.discovery.start", domain=self.domain)
+        
+        # Busca no CDX API
+        urls = await self._fetch_cdx()
+        
+        # Filtra por extensão se especificado
+        if self.filter_extensions:
+            urls = self._filter_by_extension(urls)
+        
+        # Extrai parâmetros interessantes
+        if self.extract_params:
+            self._extract_parameters(urls)
+        
+        # Detecta páginas admin/login
+        if self.detect_admin:
+            self._detect_admin_pages(urls)
+        
+        # Valida URLs ativas
+        if self.validate:
+            urls = await self._validate_urls(urls)
+        
+        logger.info("wayback.discovery.complete", 
+                   count=len(urls), 
+                   params=len(self.parameters),
+                   admin_pages=len(self.admin_urls))
+        return urls
+    
+    async def _fetch_cdx(self) -> List[WaybackURL]:
+        """Busca URLs no CDX Server."""
+        wayback_urls = []
+        
+        try:
+            async with httpx.AsyncClient(timeout=self.timeout) as client:
+                # CDX API endpoint
+                url = f"http://web.archive.org/cdx/search/cdx"
+                
+                # Monta filtro de status codes
+                status_filter = '|'.join(map(str, self.filter_status_codes))
+                
+                params = {
+                    "url": f"{self.domain}/*",
+                    "output": "json",
+                    "collapse": "urlkey",
+                    "fl": "timestamp,original,statuscode",
+                    "filter": f"statuscode:{status_filter}",
+                    "from": f"{2024 - self.years_back}",
+                }
+                
+                response = await client.get(url, params=params)
+                
+                if response.status_code == 200:
+                    data = response.json()
+                    
+                    # Primeira linha é header
+                    for entry in data[1:]:
+                        timestamp, original_url, status_code = entry
+                        
+                        wayback_url = WaybackURL(
+                            url=original_url,
+                            timestamp=timestamp,
+                            status_code=int(status_code),
+                        )
+                        wayback_urls.append(wayback_url)
+                        self.urls.add(original_url)
+                    
+                    logger.info("wayback.fetch.success", count=len(wayback_urls))
+        
+        except Exception as e:
+            logger.error("wayback.fetch.error", error=str(e))
+        
+        return wayback_urls
+    
+    def _filter_by_extension(self, urls: List[WaybackURL]) -> List[WaybackURL]:
+        """Filtra URLs por extensão."""
+        filtered = []
+        
+        for wayback_url in urls:
+            parsed = urlparse(wayback_url.url)
+            path = parsed.path.lower()
+            
+            # Verifica se tem alguma extensão desejada
+            has_extension = any(path.endswith(f".{ext}") for ext in self.filter_extensions)
+            
+            if has_extension:
+                filtered.append(wayback_url)
+        
+        logger.info("wayback.filter.complete", before=len(urls), after=len(filtered))
+        return filtered
+    
+    async def _validate_urls(self, urls: List[WaybackURL]) -> List[WaybackURL]:
+        """Valida se URLs ainda estão ativas."""
+        logger.info("wayback.validation.start", count=len(urls))
+        
+        validated = []
+        
+        async with httpx.AsyncClient(timeout=5.0, follow_redirects=True) as client:
+            tasks = [self._check_url(client, wayback_url) for wayback_url in urls]
+            
+            with Progress() as progress:
+                task_id = progress.add_task("[cyan]Validando URLs...", total=len(tasks))
+                
+                for coro in asyncio.as_completed(tasks):
+                    result = await coro
+                    if result:
+                        validated.append(result)
+                    progress.advance(task_id)
+        
+        logger.info("wayback.validation.complete", active=len(validated), total=len(urls))
+        return validated
+    
+    async def _check_url(self, client: httpx.AsyncClient, wayback_url: WaybackURL) -> Optional[WaybackURL]:
+        """Verifica se URL está ativa."""
+        try:
+            response = await client.head(wayback_url.url)
+            
+            if response.status_code < 400:
+                wayback_url.is_active = True
+                return wayback_url
+        except:
+            pass
+        
+        return None
+    
+    def _extract_parameters(self, urls: List[WaybackURL]):
+        """Extrai parâmetros interessantes das URLs."""
+        from urllib.parse import urlparse, parse_qs
+        
+        for wb_url in urls:
+            parsed = urlparse(wb_url.url)
+            if parsed.query:
+                params = parse_qs(parsed.query)
+                self.parameters.update(params.keys())
+    
+    def _detect_admin_pages(self, urls: List[WaybackURL]):
+        """Detecta páginas admin/login."""
+        admin_patterns = [
+            'admin', 'login', 'signin', 'signup', 'register',
+            'dashboard', 'panel', 'console', 'manager', 'backend',
+            'wp-admin', 'administrator', 'auth', 'user/login',
+            'account', 'portal', 'control', 'secure', 'private'
+        ]
+        
+        for wb_url in urls:
+            url_lower = wb_url.url.lower()
+            for pattern in admin_patterns:
+                if pattern in url_lower:
+                    self.admin_urls.add(wb_url.url)
+                    break
+    
+    def export(self, urls: List[WaybackURL], output: str):
+        """Exporta URLs para arquivo."""
+        with open(output, 'w') as f:
+            for wayback_url in sorted(urls, key=lambda x: x.timestamp):
+                status = "ACTIVE" if wayback_url.is_active else "ARCHIVED"
+                f.write(f"{wayback_url.timestamp} [{status}] {wayback_url.url}\n")
+        
+        logger.info("wayback.export.complete", file=output, count=len(urls))
+    
+    def export_url_list(self, output: str):
+        """Exporta lista simples de URLs para outras ferramentas."""
+        with open(output, 'w') as f:
+            for url in sorted(self.urls):
+                f.write(f"{url}\n")
+        
+        logger.info("wayback.export.urllist", file=output, count=len(self.urls))
+    
+    def export_parameters(self, output: str):
+        """Exporta lista de parâmetros encontrados."""
+        with open(output, 'w') as f:
+            for param in sorted(self.parameters):
+                f.write(f"{param}\n")
+        
+        logger.info("wayback.export.params", file=output, count=len(self.parameters))
+    
+    def export_admin_urls(self, output: str):
+        """Exporta URLs de admin/login."""
+        with open(output, 'w') as f:
+            for url in sorted(self.admin_urls):
+                f.write(f"{url}\n")
+        
+        logger.info("wayback.export.admin", file=output, count=len(self.admin_urls))
+
+
+__all__ = ["WaybackDiscovery", "WaybackURL"]

moriarty/modules/web_crawler.py

@@ -0,0 +1,163 @@
+"""Crawler HTTP leve focado em enumeração de rotas e formulários."""
+from __future__ import annotations
+
+import asyncio
+import random
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Set, TYPE_CHECKING
+
+import httpx
+from selectolax.parser import HTMLParser
+import structlog
+
+if TYPE_CHECKING:  # pragma: no cover - apenas para type hints
+    from moriarty.modules.stealth_mode import StealthMode
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class CrawlPage:
+    url: str
+    status: int
+    title: Optional[str] = None
+    forms: List[Dict[str, str]] = field(default_factory=list)
+    links: List[str] = field(default_factory=list)
+
+
+class WebCrawler:
+    """Crawler simples limitado a um domínio, ideal para pré-enumeração."""
+
+    def __init__(
+        self,
+        base_url: str,
+        max_pages: int = 100,
+        max_depth: int = 2,
+        concurrency: int = 10,
+        follow_subdomains: bool = False,
+        user_agent: str = "Mozilla/5.0 (Moriarty Recon)",
+        stealth: Optional["StealthMode"] = None,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.max_pages = max_pages
+        self.max_depth = max_depth
+        self.concurrency = concurrency
+        self.follow_subdomains = follow_subdomains
+        self.visited: Set[str] = set()
+        self.results: Dict[str, CrawlPage] = {}
+        self.stealth = stealth
+        self.user_agent = user_agent
+
+        effective_concurrency = concurrency
+        if self.stealth and getattr(self.stealth.config, "timing_randomization", False):
+            # Reduz concorrência para modos stealth altos
+            effective_concurrency = max(2, min(concurrency, int(concurrency / (self.stealth.level or 1))))
+
+        self.sem = asyncio.Semaphore(effective_concurrency)
+        self.session = httpx.AsyncClient(timeout=10.0, follow_redirects=True)
+
+        parsed = httpx.URL(self.base_url)
+        self._host = parsed.host
+        self._scheme = parsed.scheme
+
+    async def close(self) -> None:
+        await self.session.aclose()
+
+    async def crawl(self) -> Dict[str, CrawlPage]:
+        queue: asyncio.Queue = asyncio.Queue()
+        await queue.put((self.base_url, 0))
+
+        async def worker():
+            while True:
+                try:
+                    url, depth = queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    break
+                if len(self.results) >= self.max_pages or depth > self.max_depth:
+                    continue
+                if url in self.visited:
+                    continue
+                self.visited.add(url)
+                await self._fetch(url, depth, queue)
+
+        workers = [asyncio.create_task(worker()) for _ in range(self.concurrency)]
+        await asyncio.gather(*workers)
+        return self.results
+
+    async def _fetch(self, url: str, depth: int, queue: asyncio.Queue) -> None:
+        async with self.sem:
+            try:
+                await self._stealth_delay()
+                response = await self.session.get(url, headers=self._build_headers())
+            except Exception as exc:
+                logger.debug("crawler.fetch.error", url=url, error=str(exc))
+                return
+
+        page = CrawlPage(url=url, status=response.status_code)
+        if response.status_code >= 400 or not response.headers.get("content-type", "").startswith("text"):
+            self.results[url] = page
+            return
+
+        parser = HTMLParser(response.text)
+        title = parser.css_first("title")
+        page.title = title.text(strip=True) if title else None
+
+        # Forms
+        for form in parser.css("form"):
+            action = form.attributes.get("action", url)
+            method = form.attributes.get("method", "GET").upper()
+            inputs = [inp.attributes.get("name") for inp in form.css("input") if inp.attributes.get("name")]
+            page.forms.append(
+                {
+                    "action": action,
+                    "method": method,
+                    "inputs": ",".join(inputs),
+                }
+            )
+
+        # Links
+        links: Set[str] = set()
+        for anchor in parser.css("a"):
+            href = anchor.attributes.get("href")
+            if not href:
+                continue
+            href = href.strip()
+            if href.startswith("javascript:") or href.startswith("mailto:"):
+                continue
+            absolute = httpx.URL(href, base=httpx.URL(url)).human_repr()
+            if not self._should_follow(absolute):
+                continue
+            links.add(absolute)
+            if absolute not in self.visited and len(self.results) < self.max_pages:
+                await queue.put((absolute, depth + 1))
+        page.links = sorted(links)
+        self.results[url] = page
+
+    def _should_follow(self, url: str) -> bool:
+        parsed = httpx.URL(url)
+        if parsed.scheme not in {"http", "https"}:
+            return False
+        if not self.follow_subdomains and parsed.host != self._host:
+            return False
+        if not parsed.host.endswith(self._host):
+            return False
+        return True
+
+    def _build_headers(self) -> Dict[str, str]:
+        headers: Dict[str, str] = {"User-Agent": self.user_agent, "Accept": "*/*"}
+        if self.stealth:
+            stealth_headers = self.stealth.get_random_headers()
+            headers.update(stealth_headers)
+            headers.setdefault("User-Agent", stealth_headers.get("User-Agent", self.user_agent))
+        return headers
+
+    async def _stealth_delay(self) -> None:
+        if not self.stealth:
+            return
+        config = getattr(self.stealth, "config", None)
+        if not config or not getattr(config, "timing_randomization", False):
+            return
+        await asyncio.sleep(random.uniform(0.05, 0.2) * max(1, self.stealth.level))
+
+
+__all__ = ["WebCrawler", "CrawlPage"]

moriarty/net/dns_cache.py

@@ -0,0 +1,175 @@
+"""Cache DNS com suporte a TTL, persistência e métricas."""
+import asyncio
+import json
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import structlog
+
+from moriarty.core.cache_backend import (
+    BaseCacheBackend,
+    CacheRecord,
+    MemoryCacheBackend,
+    SQLiteCacheBackend,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+class DNSCache:
+    """Cache DNS thread-safe com TTL."""
+    
+    def __init__(self):
+        self._lock = asyncio.Lock()
+        self.config_manager = self._load_config_manager()
+        self.cache_config = getattr(self.config_manager, "cache", None) if self.config_manager else None
+        self.backend: BaseCacheBackend = self._initialize_backend()
+        self.metrics: Dict[str, Any] = {
+            "hits": 0,
+            "misses": 0,
+            "backend": self.backend.name,
+            "warmed": 0,
+        }
+
+        if self.cache_config and getattr(self.cache_config, "warmup_enabled", False):
+            self._warm_cache()
+    
+    async def get(self, domain: str, record_type: str) -> Optional[List[Any]]:
+        """Busca no cache."""
+        key = (domain.lower(), record_type.upper())
+        
+        async with self._lock:
+            record = self.backend.get(key)
+            if record is None:
+                logger.debug("dns.cache.miss", domain=domain, type=record_type)
+                self.metrics["misses"] += 1
+                return None
+
+            ttl_remaining = int(record.expires_at - time.time())
+            if ttl_remaining <= 0:
+                logger.debug("dns.cache.expired", domain=domain, type=record_type)
+                self.backend.delete(key)
+                self.metrics["misses"] += 1
+                return None
+
+            logger.debug(
+                "dns.cache.hit",
+                domain=domain,
+                type=record_type,
+                ttl_remaining=ttl_remaining,
+            )
+            self.metrics["hits"] += 1
+            return record.value
+    
+    async def set(self, domain: str, record_type: str, value: List[Any], ttl: int = 300):
+        """Armazena no cache."""
+        key = (domain.lower(), record_type.upper())
+        
+        async with self._lock:
+            self.backend.set(key, value, ttl)
+            logger.debug("dns.cache.set", domain=domain, type=record_type, ttl=ttl)
+    
+    async def clear(self):
+        """Limpa o cache."""
+        async with self._lock:
+            self.backend.clear()
+            logger.info("dns.cache.cleared")
+    
+    async def purge_expired(self):
+        """Remove entradas expiradas."""
+        async with self._lock:
+            self.backend.purge_expired()
+            logger.info("dns.cache.purged")
+    
+    def stats(self) -> Dict[str, Any]:
+        """Retorna estatísticas do cache."""
+        base_stats = dict(self.metrics)
+        base_stats.update(self.backend.stats())
+        return base_stats
+
+    def _load_config_manager(self):
+        try:
+            from moriarty.core.config_manager import config_manager
+
+            return config_manager
+        except Exception:
+            return None
+
+    def _initialize_backend(self) -> BaseCacheBackend:
+        backend_name = "memory"
+        max_size = 10000
+        eviction_policy = "lru"
+        sqlite_path = None
+
+        if self.cache_config:
+            backend_name = getattr(self.cache_config, "backend", "memory").lower()
+            max_size = getattr(self.cache_config, "max_size", 10000) or 10000
+            eviction_policy = getattr(self.cache_config, "eviction_policy", "lru")
+            sqlite_path = getattr(self.cache_config, "sqlite_path", None)
+
+        if backend_name == "sqlite":
+            db_path = Path(sqlite_path) if sqlite_path else self._default_sqlite_path()
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            logger.debug("dns.cache.backend", backend="sqlite", path=str(db_path))
+            return SQLiteCacheBackend(db_path, max_size)
+
+        logger.debug("dns.cache.backend", backend="memory", policy=eviction_policy)
+        return MemoryCacheBackend(max_size=max_size, eviction_policy=eviction_policy)
+
+    def _default_sqlite_path(self) -> Path:
+        if self.config_manager:
+            return self.config_manager.config_dir / "cache" / "dns_cache.db"
+        return Path.home() / ".moriarty" / "cache" / "dns_cache.db"
+
+    def _warm_cache(self) -> None:
+        if not self.config_manager:
+            return
+
+        warmup_dir = self.config_manager.config_dir / "warmup"
+        warmup_file = warmup_dir / "dns.json"
+
+        if not warmup_file.exists():
+            return
+
+        try:
+            with open(warmup_file, "r", encoding="utf-8") as handle:
+                payload = json.load(handle)
+
+            records = []
+            for item in payload.get("records", []):
+                domain = item.get("domain")
+                record_type = item.get("type", "A")
+                values = item.get("values", [])
+                ttl = int(item.get("ttl", 300))
+                if not domain or not values:
+                    continue
+                expires_at = time.time() + ttl
+                record = CacheRecord(
+                    key=(domain.lower(), record_type.upper()),
+                    value=values,
+                    ttl=ttl,
+                    expires_at=expires_at,
+                    last_access=time.time(),
+                )
+                records.append(record)
+
+            if records:
+                self.backend.warm(records)
+                self.metrics["warmed"] = len(records)
+                logger.info("dns.cache.warmed", records=len(records))
+
+        except Exception as exc:  # pragma: no cover - leitura auxiliar
+            logger.warning("dns.cache.warm_failed", error=str(exc))
+
+
+# Singleton global
+_global_cache: Optional[DNSCache] = None
+
+
+def get_global_cache() -> DNSCache:
+    """Retorna cache global."""
+    global _global_cache
+    if _global_cache is None:
+        _global_cache = DNSCache()
+    return _global_cache