moriarty-project 0.1.6__py3-none-any.whl

moriarty/net/dns_client.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from time import perf_counter
+from typing import Any, Dict, List, Optional
+
+import aiodns
+import httpx
+import structlog
+
+from .dns_cache import DNSCache, get_global_cache
+
+
+@dataclass(slots=True)
+class MXRecord:
+    priority: int
+    exchange: str
+
+
+@dataclass(slots=True)
+class TXTRecord:
+    text: str
+
+
+@dataclass(slots=True)
+class DNSLookupResult:
+    a: List[str]
+    aaaa: List[str]
+    mx: List[MXRecord]
+    txt: List[TXTRecord]
+    spf: List[str]
+    dmarc: List[str]
+
+
+class DNSClient:
+    def __init__(
+        self,
+        timeout: float = 8.0,
+        use_cache: bool = True,
+        doh_endpoint: Optional[str] = None,
+        dot_server: Optional[str] = None,
+    ) -> None:
+        self._resolver = aiodns.DNSResolver(timeout=timeout)
+        self._timeout = timeout
+        self._logger = structlog.get_logger(__name__)
+        self._use_cache = use_cache
+        self._cache = get_global_cache() if use_cache else None
+        self._doh_endpoint = doh_endpoint  # Ex: https://cloudflare-dns.com/dns-query
+        self._dot_server = dot_server  # Ex: 1.1.1.1:853
+        
+        if doh_endpoint:
+            self._logger.info("dns.doh.enabled", endpoint=doh_endpoint)
+        if dot_server:
+            self._logger.info("dns.dot.enabled", server=dot_server)
+
+    async def lookup_domain(self, domain: str) -> DNSLookupResult:
+        start = perf_counter()
+        self._logger.info("dns.lookup.start", domain=domain, timeout_s=self._timeout)
+        a_task = asyncio.create_task(self._query(domain, "A"))
+        aaaa_task = asyncio.create_task(self._query(domain, "AAAA"))
+        mx_task = asyncio.create_task(self._query(domain, "MX"))
+        txt_task = asyncio.create_task(self._query(domain, "TXT"))
+        dmarc_task = asyncio.create_task(self._query(f"_dmarc.{domain}", "TXT"))
+
+        a_records = await a_task
+        aaaa_records = await aaaa_task
+        mx_records = await mx_task
+        txt_records = await txt_task
+        dmarc_records = await dmarc_task
+
+        mx_structured = [
+            MXRecord(priority=int(entry.get("priority", 0)), exchange=str(entry.get("host")))
+            for entry in mx_records
+            if "host" in entry
+        ]
+
+        txt_structured = [
+            TXTRecord(text=str(entry.get("text", "")))
+            for entry in txt_records
+            if entry.get("text")
+        ]
+
+        spf_records = [record.text for record in txt_structured if record.text.lower().startswith("v=spf1")]
+        dmarc_structured = [
+            TXTRecord(text=str(entry.get("text", "")))
+            for entry in dmarc_records
+            if entry.get("text")
+        ]
+
+        result = DNSLookupResult(
+            a=[entry.get("host") for entry in a_records if entry.get("host")],
+            aaaa=[entry.get("host") for entry in aaaa_records if entry.get("host")],
+            mx=sorted(mx_structured, key=lambda mx: (mx.priority, mx.exchange)),
+            txt=txt_structured,
+            spf=spf_records,
+            dmarc=[record.text for record in dmarc_structured],
+        )
+        latency_ms = (perf_counter() - start) * 1000
+        self._logger.info(
+            "dns.lookup.success",
+            domain=domain,
+            latency_ms=round(latency_ms, 2),
+            a=len(result.a),
+            aaaa=len(result.aaaa),
+            mx=len(result.mx),
+        )
+        return result
+
+    async def _query(self, domain: str, record_type: str) -> List[Dict[str, Any]]:
+        # Verifica cache primeiro
+        if self._cache:
+            cached = await self._cache.get(domain, record_type)
+            if cached is not None:
+                return cached
+        
+        # DoH (DNS-over-HTTPS)
+        if self._doh_endpoint:
+            try:
+                result = await self._query_doh(domain, record_type)
+                if result and self._cache:
+                    await self._cache.set(domain, record_type, result, ttl=300)
+                return result
+            except Exception as e:
+                self._logger.warning("dns.doh.error", error=str(e), domain=domain)
+                # Fallback para DNS tradicional
+        
+        # DNS tradicional
+        try:
+            result = await self._resolver.query(domain, record_type)
+        except aiodns.error.DNSError:
+            self._logger.warning("dns.lookup.miss", domain=domain, record_type=record_type)
+            return []
+
+        if isinstance(result, list):
+            records = [self._record_to_dict(entry) for entry in result]
+            
+            # Salva no cache com TTL
+            if self._cache and records:
+                ttl = getattr(result[0], 'ttl', 300) if result else 300
+                await self._cache.set(domain, record_type, records, ttl=ttl)
+            
+            return records
+
+        return []
+    
+    async def _query_doh(self, domain: str, record_type: str) -> List[Dict[str, Any]]:
+        """Consulta DNS via HTTPS (DoH)."""
+        async with httpx.AsyncClient(timeout=self._timeout) as client:
+            response = await client.get(
+                self._doh_endpoint,
+                params={
+                    "name": domain,
+                    "type": record_type,
+                },
+                headers={"Accept": "application/dns-json"},
+            )
+            response.raise_for_status()
+            data = response.json()
+            
+            # Converte resposta DoH para formato compatível
+            answers = data.get("Answer", [])
+            if not answers:
+                return []
+            
+            records = []
+            for answer in answers:
+                if record_type == "A" or record_type == "AAAA":
+                    records.append({"host": answer.get("data"), "ttl": answer.get("TTL", 300)})
+                elif record_type == "MX":
+                    parts = answer.get("data", "").split()
+                    if len(parts) == 2:
+                        records.append({"priority": int(parts[0]), "host": parts[1], "ttl": answer.get("TTL", 300)})
+                elif record_type == "TXT":
+                    records.append({"text": answer.get("data", ""), "ttl": answer.get("TTL", 300)})
+            
+            return records
+
+    @staticmethod
+    def _record_to_dict(record: Any) -> Dict[str, Any]:
+        return {
+            key: getattr(record, key)
+            for key in dir(record)
+            if not key.startswith("_") and not callable(getattr(record, key))
+        }
+
+
+__all__ = ["DNSClient", "DNSLookupResult", "MXRecord", "TXTRecord"]

moriarty/net/rdap_client.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from time import perf_counter
+from typing import Any, Dict
+
+import httpx
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass(slots=True)
+class RDAPResponse:
+    url: str
+    payload: Dict[str, Any]
+    status: int
+    latency_ms: float
+
+
+class RDAPClient:
+    def __init__(self, base_url: str = "https://rdap.org", timeout: float = 8.0, http2: bool = True) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+        self._http2 = http2
+
+    async def fetch(self, path: str) -> RDAPResponse:
+        url = f"{self._base_url}/{path.lstrip('/')}"
+        logger.info("rdap.request.start", url=url)
+        start = perf_counter()
+        async with httpx.AsyncClient(timeout=self._timeout, http2=self._http2) as client:
+            response = await client.get(url, headers={"Accept": "application/rdap+json, application/json"})
+        latency_ms = (perf_counter() - start) * 1000
+        if response.status_code >= 400:
+            logger.warning(
+                "rdap.request.error",
+                url=url,
+                status=response.status_code,
+                latency_ms=round(latency_ms, 2),
+            )
+            response.raise_for_status()
+        payload = response.json()
+        logger.info(
+            "rdap.request.success",
+            url=url,
+            status=response.status_code,
+            latency_ms=round(latency_ms, 2),
+        )
+        return RDAPResponse(url=url, payload=payload, status=response.status_code, latency_ms=latency_ms)
+
+
+__all__ = ["RDAPClient", "RDAPResponse"]

moriarty/net/smtp_client.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from dataclasses import dataclass
+from time import perf_counter
+from typing import Iterable, List, Optional
+
+import aiosmtplib
+import structlog
+
+
+@dataclass(slots=True)
+class SMTPAttempt:
+    host: str
+    port: int
+    result_code: Optional[int]
+    result_message: Optional[str]
+    success: bool
+    error: Optional[str]
+
+
+@dataclass(slots=True)
+class SMTPProbeResult:
+    deliverable: bool
+    attempts: List[SMTPAttempt]
+
+
+class SMTPClient:
+    def __init__(self, timeout: float = 8.0, wait: float = 1.0, retries: int = 1) -> None:
+        self._timeout = timeout
+        self._wait = wait
+        self._retries = retries
+        self._logger = structlog.get_logger(__name__)
+
+    async def probe(self, email: str, from_address: str, hosts: Iterable[str]) -> SMTPProbeResult:
+        attempts: List[SMTPAttempt] = []
+        deliverable = False
+
+        for host in hosts:
+            start = perf_counter()
+            self._logger.info(
+                "smtp.probe.start",
+                host=host,
+                retries=self._retries,
+                timeout_s=self._timeout,
+            )
+            for attempt in range(self._retries + 1):
+                smtp: Optional[aiosmtplib.SMTP] = None
+                try:
+                    smtp = aiosmtplib.SMTP(hostname=host, port=25, timeout=self._timeout)
+                    await smtp.connect()
+                    await smtp.ehlo()
+                    await smtp.mail(from_address)
+                    rcpt_code, rcpt_message = await smtp.rcpt(email)
+                    message_text = rcpt_message.decode("utf-8", errors="replace") if isinstance(rcpt_message, bytes) else str(rcpt_message)
+                    success = 200 <= rcpt_code < 300
+                    attempts.append(
+                        SMTPAttempt(
+                            host=host,
+                            port=25,
+                            result_code=rcpt_code,
+                            result_message=message_text,
+                            success=success,
+                            error=None,
+                        )
+                    )
+                    if success:
+                        deliverable = True
+                        await smtp.quit()
+                        return SMTPProbeResult(deliverable=deliverable, attempts=attempts)
+                except aiosmtplib.errors.SMTPException as exc:
+                    attempts.append(
+                        SMTPAttempt(
+                            host=host,
+                            port=25,
+                            result_code=None,
+                            result_message=None,
+                            success=False,
+                            error=str(exc),
+                        )
+                    )
+                except Exception as exc:  # noqa: BLE001
+                    attempts.append(
+                        SMTPAttempt(
+                            host=host,
+                            port=25,
+                            result_code=None,
+                            result_message=None,
+                            success=False,
+                            error=str(exc),
+                        )
+                    )
+                finally:
+                    if smtp is not None:
+                        with contextlib.suppress(Exception):
+                            await smtp.quit()
+
+                    if attempt < self._retries:
+                        await asyncio.sleep(self._wait)
+
+            latency_ms = (perf_counter() - start) * 1000
+            self._logger.info(
+                "smtp.probe.complete",
+                host=host,
+                attempts=len(attempts),
+                latency_ms=round(latency_ms, 2),
+                deliverable=deliverable,
+            )
+
+        return SMTPProbeResult(deliverable=deliverable, attempts=attempts)
+
+
+__all__ = ["SMTPClient", "SMTPProbeResult", "SMTPAttempt"]

moriarty/net/tls_client.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import socket
+import ssl
+from dataclasses import dataclass
+from datetime import datetime
+from time import perf_counter
+from typing import List, Optional
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass(slots=True)
+class TLSCertificate:
+    subject: str
+    issuer: str
+    not_before: datetime
+    not_after: datetime
+    fingerprint_sha256: str
+    subject_alt_names: List[str]
+
+
+@dataclass(slots=True)
+class TLSInspection:
+    host: str
+    port: int
+    protocol: Optional[str]
+    cipher: Optional[str]
+    certificates: List[TLSCertificate]
+    latency_ms: float
+
+
+class TLSClient:
+    def __init__(self, timeout: float = 8.0, verify: bool = False) -> None:
+        self._timeout = timeout
+        self._verify = verify
+
+    async def inspect(self, host: str, port: int = 443) -> TLSInspection:
+        logger.info("tls.inspect.start", host=host, port=port)
+        start = perf_counter()
+        try:
+            protocol, cipher, certificates = await asyncio.wait_for(
+                asyncio.to_thread(self._inspect_sync, host, port),
+                timeout=self._timeout,
+            )
+        except Exception:
+            logger.exception("tls.inspect.error", host=host, port=port)
+            raise
+        latency_ms = (perf_counter() - start) * 1000
+        logger.info(
+            "tls.inspect.success",
+            host=host,
+            port=port,
+            latency_ms=round(latency_ms, 2),
+            certificates=len(certificates),
+        )
+        return TLSInspection(
+            host=host,
+            port=port,
+            protocol=protocol,
+            cipher=cipher,
+            certificates=certificates,
+            latency_ms=latency_ms,
+        )
+
+    def _inspect_sync(self, host: str, port: int) -> tuple[Optional[str], Optional[str], List[TLSCertificate]]:
+        context = ssl.create_default_context()
+        if not self._verify:
+            context.check_hostname = False
+            context.verify_mode = ssl.CERT_NONE
+
+        with socket.create_connection((host, port), timeout=self._timeout) as sock:
+            with context.wrap_socket(sock, server_hostname=host) as tls_sock:
+                cipher_info = tls_sock.cipher()
+                protocol = tls_sock.version()
+                der_cert = tls_sock.getpeercert(binary_form=True)
+                cert_dict = tls_sock.getpeercert()
+
+        certificate = self._parse_cert(der_cert, cert_dict)
+        return protocol, cipher_info[0] if cipher_info else None, [certificate]
+
+    def _parse_cert(self, der_bytes: bytes, cert_dict: dict[str, object]) -> TLSCertificate:
+        fingerprint = hashlib.sha256(der_bytes).hexdigest()
+        subject_components = cert_dict.get("subject", [])
+        subject = ", ".join("=".join(component) for rdn in subject_components for component in rdn)
+        issuer_components = cert_dict.get("issuer", [])
+        issuer = ", ".join("=".join(component) for rdn in issuer_components for component in rdn)
+        not_before = self._parse_time(cert_dict.get("notBefore"))
+        not_after = self._parse_time(cert_dict.get("notAfter"))
+        sans = [value for key, value in cert_dict.get("subjectAltName", []) if value]
+        return TLSCertificate(
+            subject=subject,
+            issuer=issuer,
+            not_before=not_before,
+            not_after=not_after,
+            fingerprint_sha256=fingerprint,
+            subject_alt_names=sans,
+        )
+
+    @staticmethod
+    def _parse_time(value: Optional[object]) -> datetime:
+        if isinstance(value, str):
+            return datetime.strptime(value, "%b %d %H:%M:%S %Y %Z")
+        return datetime.fromtimestamp(0)
+
+
+__all__ = ["TLSClient", "TLSInspection", "TLSCertificate"]

moriarty/parsers/html_parser.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import re
+from typing import Any, List, Optional
+
+from selectolax.parser import HTMLParser
+
+from ..dsl.schema import SelectorSpec
+
+
+class ParserError(Exception):
+    """Erro ao fazer parsing."""
+
+
+class HTMLExtractor:
+    """Extrai dados de HTML usando selectores."""
+
+    def __init__(self, html: str) -> None:
+        self._parser = HTMLParser(html)
+        self._raw_html = html
+
+    def extract(self, selector: SelectorSpec) -> Optional[str]:
+        """Extrai um único valor usando um seletor."""
+        if selector.type == "css":
+            return self._extract_css(selector)
+        elif selector.type == "xpath":
+            # selectolax não suporta XPath nativamente, fallback para lxml se necessário
+            return self._extract_xpath(selector)
+        elif selector.type == "regex":
+            return self._extract_regex(selector)
+        elif selector.type == "json":
+            # JSON path sobre o HTML bruto
+            return None
+        return None
+
+    def extract_all(self, selector: SelectorSpec) -> List[str]:
+        """Extrai múltiplos valores."""
+        if selector.type == "css":
+            return self._extract_css_all(selector)
+        elif selector.type == "regex":
+            return self._extract_regex_all(selector)
+        return []
+
+    def exists(self, selector: SelectorSpec) -> bool:
+        """Verifica se um seletor encontra algo."""
+        result = self.extract(selector)
+        return result is not None and len(result) > 0
+
+    def _extract_css(self, selector: SelectorSpec) -> Optional[str]:
+        """Extrai usando CSS selector."""
+        node = self._parser.css_first(selector.query)
+        if node is None:
+            return None
+
+        if selector.attribute:
+            return node.attributes.get(selector.attribute)
+        
+        return node.text(strip=True)
+
+    def _extract_css_all(self, selector: SelectorSpec) -> List[str]:
+        """Extrai todos os matches de um CSS selector."""
+        nodes = self._parser.css(selector.query)
+        results: List[str] = []
+
+        for node in nodes:
+            if selector.attribute:
+                value = node.attributes.get(selector.attribute)
+                if value:
+                    results.append(value)
+            else:
+                text = node.text(strip=True)
+                if text:
+                    results.append(text)
+
+        return results
+
+    def _extract_xpath(self, selector: SelectorSpec) -> Optional[str]:
+        """Extrai usando XPath (requer lxml)."""
+        try:
+            from lxml import etree, html
+        except ImportError:
+            raise ParserError("lxml required for XPath selectors")
+
+        try:
+            tree = html.fromstring(self._raw_html)
+            result = tree.xpath(selector.query)
+            
+            if not result:
+                return None
+            
+            if isinstance(result, list) and len(result) > 0:
+                element = result[0]
+                if selector.attribute and hasattr(element, "get"):
+                    return element.get(selector.attribute)
+                if hasattr(element, "text"):
+                    return element.text
+                return str(element)
+            
+            return str(result)
+        except Exception:
+            return None
+
+    def _extract_regex(self, selector: SelectorSpec) -> Optional[str]:
+        """Extrai usando regex."""
+        pattern = re.compile(selector.query, re.IGNORECASE | re.DOTALL)
+        match = pattern.search(self._raw_html)
+        
+        if match:
+            group = selector.group if selector.group is not None else 0
+            try:
+                return match.group(group)
+            except IndexError:
+                return None
+        
+        return None
+
+    def _extract_regex_all(self, selector: SelectorSpec) -> List[str]:
+        """Extrai todos os matches de regex."""
+        pattern = re.compile(selector.query, re.IGNORECASE | re.DOTALL)
+        matches = pattern.finditer(self._raw_html)
+        
+        results: List[str] = []
+        group = selector.group if selector.group is not None else 0
+        
+        for match in matches:
+            try:
+                value = match.group(group)
+                if value:
+                    results.append(value)
+            except IndexError:
+                continue
+        
+        return results
+
+
+__all__ = ["HTMLExtractor", "ParserError"]

moriarty/tests/test_email_service.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import asyncio
+
+from moriarty.modules.email_check import EmailCheckOptions, EmailCheckService
+
+
+def test_email_normalization_only() -> None:
+    service = EmailCheckService(
+        email="Alice@Example.COM",
+        options=EmailCheckOptions(check_dns=False, check_smtp=False),
+        timeout=0.1,
+    )
+    result = asyncio.run(service.run())
+    assert result.normalized_email == "alice@example.com"
+    assert result.dns is None
+    assert result.smtp is None

moriarty/tests/test_models.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from moriarty.models import (
+    Assertion,
+    ConfidenceBand,
+    CustodyEvent,
+    Entity,
+    EntityKind,
+    Evidence,
+    EvidenceKind,
+    Relation,
+)
+
+
+def test_entity_creation() -> None:
+    entity = Entity(kind=EntityKind.EMAIL, value="alice@example.com")
+    assert entity.value == "alice@example.com"
+    assert entity.kind is EntityKind.EMAIL
+
+
+def test_evidence_chain() -> None:
+    evidence = Evidence(
+        kind=EvidenceKind.NETWORK,
+        sha256="deadbeef",
+        source="dns",
+        custody=[CustodyEvent(actor="tester", action="collected")],
+    )
+    assert evidence.custody[0].actor == "tester"
+
+
+def test_assertion_defaults() -> None:
+    entity = Entity(kind=EntityKind.DOMAIN, value="example.com")
+    assertion = Assertion(
+        subject_id=entity.id,
+        predicate="resolves_to",
+        object="93.184.216.34",
+    )
+    assert assertion.band == ConfidenceBand.INDICATIVE
+
+
+def test_relation_attributes() -> None:
+    src = Entity(kind=EntityKind.EMAIL, value="alice@example.com")
+    dst = Entity(kind=EntityKind.USERNAME, value="alice")
+    relation = Relation(source_id=src.id, target_id=dst.id, relation_type="aka")
+    assert relation.source_id == src.id
+    assert relation.relation_type == "aka"