detecti-cli 2.0.12__tar.gz → 2.1.0__tar.gz

{detecti_cli-2.0.12 → detecti_cli-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: detecti-cli
-Version: 2.0.12
+Version: 2.1.0
 Summary: Modern Attack Surface Management & Threat Intelligence Engine
 Author-email: "Lucas S. (Ls4ss)" <lucas@lucassouza.io>
 License: MIT

{detecti_cli-2.0.12 → detecti_cli-2.1.0}/detecti/cli.py

@@ -264,6 +264,18 @@ def scan_command(
             try:
                 progress.update(task_id, description=f"[bold cyan]Storing results in SQLite database ({final_db_name})...")
                 db_manager.store_scan_result(result)
+                
+                # Lazy Summary: Sync the CLI Executive Summary with the Database Post-Gating metrics
+                stats = db_manager.get_summary_stats()
+                if result.summary:
+                    result.summary.total_hosts_count = stats.get('total_ips', result.summary.total_hosts_count)
+                    result.summary.subdomains_count = stats.get('total_subdomains', result.summary.subdomains_count)
+                    # For associated domains in UI we refer to total_domains
+                    result.summary.associated_domains_count = stats.get('total_domains', result.summary.associated_domains_count)
+                    result.summary.open_ports_count = stats.get('open_services', result.summary.open_ports_count)
+                    result.summary.vulnerabilities_count = stats.get('total_vulnerabilities', result.summary.vulnerabilities_count)
+                    result.summary.cisa_kev_count = stats.get('cisa_kev_count', result.summary.cisa_kev_count)
+
                 print_success(f"Scan results stored in database: [bold underline]{db_manager.db_path.resolve() if hasattr(db_manager.db_path, 'resolve') else db_manager.db_path}[/bold underline]")
             except Exception as e:
                 print_error(f"Failed to store results in database: {e}")

{detecti_cli-2.0.12 → detecti_cli-2.1.0}/detecti/core/database/schema.py

@@ -38,6 +38,7 @@ CREATE TABLE IF NOT EXISTS ip_addresses (
 CREATE TABLE IF NOT EXISTS subdomain_ips (
     subdomain_id TEXT NOT NULL,
     ip_id TEXT NOT NULL,
+    resolution_type TEXT DEFAULT 'RESOLVES_TO',
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (subdomain_id, ip_id),
     FOREIGN KEY (subdomain_id) REFERENCES subdomains(id),
@@ -48,6 +49,7 @@ CREATE TABLE IF NOT EXISTS subdomain_ips (
 CREATE TABLE IF NOT EXISTS services (
     id TEXT PRIMARY KEY,
     ip_id TEXT NOT NULL,
+    resolution_type TEXT DEFAULT 'RESOLVES_TO',
     port INTEGER NOT NULL,
     protocol TEXT DEFAULT 'tcp',
     service_name TEXT,

{detecti_cli-2.0.12 → detecti_cli-2.1.0}/detecti/core/database/storage.py

@@ -89,6 +89,22 @@ class DatabaseManager:
             except Exception:
                 pass
 
+            # Auto-migrate: ensure metadata column exists in subdomains table for WAF bypass tags
+            try:
+                sub_cols = [row[1] for row in conn.execute("PRAGMA table_info(subdomains)").fetchall()]
+                if "metadata" not in sub_cols:
+                    conn.execute("ALTER TABLE subdomains ADD COLUMN metadata TEXT")
+            except Exception:
+                pass
+            # Auto-migrate: ensure resolution_type exists in subdomain_ips table
+            try:
+                sub_ip_cols = [row[1] for row in conn.execute("PRAGMA table_info(subdomain_ips)").fetchall()]
+                if "resolution_type" not in sub_ip_cols:
+                    conn.execute("ALTER TABLE subdomain_ips ADD COLUMN resolution_type TEXT DEFAULT 'RESOLVES_TO'")
+            except Exception:
+                pass
+
+
             # Auto-clean: deduplicate any existing redundant services per (ip_id, port, protocol)
             self._deduplicate_services(conn)
                 
@@ -224,7 +240,7 @@ class DatabaseManager:
         subdomain_map = {}
         
         # Helper to register any candidate subdomain
-        def _register_subdomain_candidate(raw_name: str) -> None:
+        def _register_subdomain_candidate(raw_name: str, meta: Optional[Dict] = None) -> None:
             if not raw_name:
                 return
             cand = raw_name.strip().lower()
@@ -276,12 +292,20 @@ class DatabaseManager:
                             INSERT INTO subdomains (id, domain_id, name)
                             VALUES (?, ?, ?)
                         """, (subdomain_id, domain_id, cand))
+                    
+                    if meta:
+                        try:
+                            meta_json = json.dumps(meta)
+                            conn.execute("UPDATE subdomains SET metadata = ? WHERE id = ?", (meta_json, subdomain_id))
+                        except Exception:
+                            pass
+
                     subdomain_map[cand] = subdomain_id
 
         # 1. Register subdomains from FindingType.SUBDOMAIN, ASSOCIATED_DOMAIN and targets
         for finding in findings:
             if finding.type in (FindingType.SUBDOMAIN, FindingType.ASSOCIATED_DOMAIN) and finding.value:
-                _register_subdomain_candidate(finding.value)
+                _register_subdomain_candidate(finding.value, finding.metadata)
             if finding.target:
                 _register_subdomain_candidate(finding.target)
             if finding.type == FindingType.HOST_INFO and finding.host_info:
@@ -295,10 +319,10 @@ class DatabaseManager:
             for host in hosts:
                 if host.hostnames:
                     for hname in host.hostnames:
-                        _register_subdomain_candidate(hname)
+                        _register_subdomain_candidate(hname, host.metadata if host.metadata.get("waf_bypassed_domains") and hname in host.metadata["waf_bypassed_domains"] else None)
                 if host.domains:
                     for dname in host.domains:
-                        _register_subdomain_candidate(dname)
+                        _register_subdomain_candidate(dname, host.metadata if host.metadata.get("waf_bypassed_domains") and dname in host.metadata["waf_bypassed_domains"] else None)
         
         return subdomain_map
 
@@ -547,10 +571,13 @@ class DatabaseManager:
                         if hname_clean.startswith("*."):
                             hname_clean = hname_clean[2:]
                         if hname_clean in subdomain_map:
+                            res_type = "RESOLVES_TO" if any("dns" in s.lower() and "historical" not in s.lower() for s in getattr(host, "sources", [])) else "IPS_HISTORY"
                             conn.execute("""
-                                INSERT OR IGNORE INTO subdomain_ips (subdomain_id, ip_id)
-                                VALUES (?, ?)
-                            """, (subdomain_map[hname_clean], ip_id))
+                                INSERT INTO subdomain_ips (subdomain_id, ip_id, resolution_type)
+                                VALUES (?, ?, ?)
+                                ON CONFLICT (subdomain_id, ip_id) DO UPDATE SET 
+                                resolution_type = CASE WHEN excluded.resolution_type = 'RESOLVES_TO' THEN 'RESOLVES_TO' ELSE subdomain_ips.resolution_type END
+                            """, (subdomain_map[hname_clean], ip_id, res_type))
             
             # Map specific authoritative DNS resolutions & finding associations (subdomain -> IP)
             for finding in result.findings:
@@ -565,10 +592,13 @@ class DatabaseManager:
                             if hname_clean.startswith("*."):
                                 hname_clean = hname_clean[2:]
                             if hname_clean in subdomain_map:
+                                res_type = "RESOLVES_TO" if ("dns" in getattr(finding, "source", "").lower() and "historical" not in getattr(finding, "source", "").lower()) else "IPS_HISTORY"
                                 conn.execute("""
-                                    INSERT OR IGNORE INTO subdomain_ips (subdomain_id, ip_id)
-                                    VALUES (?, ?)
-                                """, (subdomain_map[hname_clean], target_ip_id))
+                                    INSERT INTO subdomain_ips (subdomain_id, ip_id, resolution_type)
+                                    VALUES (?, ?, ?)
+                                    ON CONFLICT (subdomain_id, ip_id) DO UPDATE SET 
+                                    resolution_type = CASE WHEN excluded.resolution_type = 'RESOLVES_TO' THEN 'RESOLVES_TO' ELSE subdomain_ips.resolution_type END
+                                """, (subdomain_map[hname_clean], target_ip_id, res_type))
                     
                     # 2. Subdomain finding with explicit host_ip
                     if finding.type == FindingType.SUBDOMAIN and finding.value and finding.host_ip == hip:
@@ -576,10 +606,13 @@ class DatabaseManager:
                         if sub_val.startswith("*."):
                             sub_val = sub_val[2:]
                         if sub_val in subdomain_map:
+                            res_type = "RESOLVES_TO" if ("dns" in getattr(finding, "source", "").lower() and "historical" not in getattr(finding, "source", "").lower()) else "IPS_HISTORY"
                             conn.execute("""
-                                INSERT OR IGNORE INTO subdomain_ips (subdomain_id, ip_id)
-                                VALUES (?, ?)
-                            """, (subdomain_map[sub_val], target_ip_id))
+                                INSERT INTO subdomain_ips (subdomain_id, ip_id, resolution_type)
+                                VALUES (?, ?, ?)
+                                ON CONFLICT (subdomain_id, ip_id) DO UPDATE SET 
+                                resolution_type = CASE WHEN excluded.resolution_type = 'RESOLVES_TO' THEN 'RESOLVES_TO' ELSE subdomain_ips.resolution_type END
+                            """, (subdomain_map[sub_val], target_ip_id, res_type))
 
             conn.commit()
 
@@ -927,8 +960,9 @@ class DatabaseManager:
 
                     # Ensure direct link between FQDN and IP
                     conn.execute("""
-                        INSERT OR IGNORE INTO subdomain_ips (subdomain_id, ip_id)
-                        VALUES (?, ?)
+                        INSERT INTO subdomain_ips (subdomain_id, ip_id, resolution_type)
+                        VALUES (?, ?, 'RESOLVES_TO')
+                        ON CONFLICT (subdomain_id, ip_id) DO UPDATE SET resolution_type = 'RESOLVES_TO'
                     """, (sub_id, cur_ip_id))
 
                 # 4. If DNS resolution was offline/empty, fallback to any existing database links
@@ -1219,7 +1253,7 @@ class DatabaseManager:
                                         d_row = conn.execute("SELECT id FROM domains LIMIT 1").fetchone()
                                         d_id = d_row[0] if d_row else None
                                         conn.execute("INSERT OR IGNORE INTO subdomains (id, domain_id, name) VALUES (?, ?, ?)", (s_id, d_id, clean_candidate))
-                                    conn.execute("INSERT OR IGNORE INTO subdomain_ips (subdomain_id, ip_id) VALUES (?, ?)", (s_id, ip_id))
+                                    conn.execute("INSERT INTO subdomain_ips (subdomain_id, ip_id, resolution_type) VALUES (?, ?, 'RESOLVES_TO') ON CONFLICT (subdomain_id, ip_id) DO UPDATE SET resolution_type = 'RESOLVES_TO'", (s_id, ip_id))
                             except Exception:
                                 pass
 

{detecti_cli-2.0.12 → detecti_cli-2.1.0}/detecti/core/engine.py

@@ -436,6 +436,12 @@ class ThreatTrackEngine:
                 self._notify("reverse_whois", f"Performing Reverse WHOIS lookup for {query_whois}...")
                 recon_tasks.append(self.modules["reverse_whois"].run(query_whois, context))
 
+            # 5. SecurityTrails
+            if target_type in ("domain", "ip") and "securitytrails" in active_mod_names:
+                st_target = root_domain or clean_target if target_type == "domain" else clean_target
+                self._notify("securitytrails", f"Querying SecurityTrails for {st_target}...")
+                recon_tasks.append(self.modules["securitytrails"].run(st_target, context))
+
         if recon_tasks:
             recon_results = await asyncio.gather(*recon_tasks, return_exceptions=True)
             for res in recon_results:
@@ -509,12 +515,14 @@ class ThreatTrackEngine:
 
             for sub, ips in resolve_results:
                 if ips:
-                    # In an IP scan, if the resolved IPs do NOT include the target IP, this domain has migrated elsewhere
+                    # In an IP scan, if the resolved IPs do NOT include the target IP, this domain has migrated elsewhere or is behind a WAF
                     if is_ip_scan and target_ip_clean and target_ip_clean not in ips:
-                        # Drop foreign domain findings to keep attack surface strict and prevent third-party noise
+                        # Tag it as WAF Bypass / Origin Match instead of dropping
                         for sub_finding in subdomain_finding_refs.get(sub, []):
                             if sub_finding in raw_recon_findings:
-                                raw_recon_findings.remove(sub_finding)
+                                sub_finding.metadata["is_waf_bypass"] = True
+                                # Anchor the finding's host_ip to the TARGET IP (the origin), NOT the CDN IP!
+                                sub_finding.host_ip = target_ip_clean
                         continue
 
                     for ip in ips:
@@ -726,10 +734,18 @@ class ThreatTrackEngine:
                 elif f.type == FindingType.SUBDOMAIN and f.value:
                     if f.value not in host_obj.hostnames:
                         host_obj.hostnames.append(f.value)
+                    if f.metadata.get("is_waf_bypass"):
+                        waf_list = host_obj.metadata.setdefault("waf_bypassed_domains", [])
+                        if f.value not in waf_list:
+                            waf_list.append(f.value)
 
                 elif f.type == FindingType.ASSOCIATED_DOMAIN and f.value:
                     if f.value not in host_obj.domains:
                         host_obj.domains.append(f.value)
+                    if f.metadata.get("is_waf_bypass"):
+                        waf_list = host_obj.metadata.setdefault("waf_bypassed_domains", [])
+                        if f.value not in waf_list:
+                            waf_list.append(f.value)
 
         # ----------------------------------------------------
         # Ensure Target Anchor in Hosts Map & Domain Discoveries

detecti_cli-2.1.0/detecti/modules/securitytrails.py

@@ -0,0 +1,267 @@
+"""SecurityTrails advanced intelligence module (DSL & Anti-WAF)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional
+
+from detecti.config import settings
+from detecti.core.models import Finding, FindingType, HostInfoData
+from detecti.modules.base import BaseModule
+
+logger = logging.getLogger("detecti.securitytrails")
+
+
+class SecurityTrailsModule(BaseModule):
+    """Smart SecurityTrails module for comprehensive attack surface pivoting."""
+
+    name: str = "securitytrails"
+    description: str = "SecurityTrails API for DSL intelligence, WHOIS pivoting, and Anti-WAF Origin IP discovery"
+    category: str = "recon"
+
+    def is_configured(self) -> bool:
+        """Check if valid SecurityTrails API key is set."""
+        from detecti.config import is_placeholder_key
+        return bool(settings.securitytrails_api_key and not is_placeholder_key(settings.securitytrails_api_key))
+
+    async def _scroll_dsl_query(self, query: str, max_pages: int = 5) -> List[str]:
+        """Fetch domains using the SecurityTrails DSL search list API with pagination."""
+        headers = {
+            "APIKEY": settings.securitytrails_api_key,
+            "Accept": "application/json"
+        }
+        url = "https://api.securitytrails.com/v1/domains/list"
+        
+        all_domains = []
+        page = 1
+        
+        while page <= max_pages:
+            payload = {"query": query}
+            
+            try:
+                # Note: The DSL API uses URL query params for pagination, e.g., ?page=1
+                page_url = f"{url}?page={page}"
+                resp = await self.http_client.post(
+                    url=page_url,
+                    headers=headers,
+                    json=payload,
+                    timeout=15.0,
+                    max_retries=2,
+                )
+                
+                # If 404/403, might be wrong endpoint or permissions. We'll fallback to search/list if domains/list fails.
+                if resp.status_code == 404:
+                    page_url = f"https://api.securitytrails.com/v1/search/list?page={page}"
+                    resp = await self.http_client.post(
+                        url=page_url,
+                        headers=headers,
+                        json=payload,
+                        timeout=15.0,
+                        max_retries=2,
+                        raise_for_status=True
+                    )
+                else:
+                    resp.raise_for_status()
+
+                data = resp.json()
+                records = data.get("records", [])
+                for rec in records:
+                    hostname = rec.get("hostname")
+                    if hostname:
+                        all_domains.append(hostname)
+                
+                meta = data.get("meta", {})
+                total_pages = meta.get("max_page", 1)
+                
+                if page >= total_pages:
+                    break
+                    
+                page += 1
+            except Exception as e:
+                logger.debug(f"SecurityTrails DSL error on page {page}: {e}")
+                if hasattr(e, "response") and e.response is not None and e.response.status_code == 403:
+                    self.notify("SecurityTrails API: DSL feature requires a commercial subscription (403 Forbidden).")
+                elif page == 1:
+                    self.notify(f"SecurityTrails API error: {e}")
+                break
+                
+        return all_domains
+
+    async def _fetch_historical_dns(self, domain: str) -> List[str]:
+        """Fetch historical A records to find potential Origin IPs (WAF Bypass)."""
+        headers = {
+            "APIKEY": settings.securitytrails_api_key,
+            "Accept": "application/json"
+        }
+        url = f"https://api.securitytrails.com/v1/history/{domain}/dns/a"
+        ips = set()
+        
+        try:
+            resp = await self.http_client.get(
+                url=url,
+                headers=headers,
+                timeout=10.0,
+                max_retries=1
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                records = data.get("records", [])
+                for rec in records:
+                    for val in rec.get("values", []):
+                        ip = val.get("ip")
+                        if ip:
+                            ips.add(ip)
+        except Exception as e:
+            logger.debug(f"Historical DNS error for {domain}: {e}")
+            
+        return list(ips)
+
+    async def _fetch_subdomains(self, domain: str) -> List[str]:
+        """Fetch subdomains for an apex domain."""
+        headers = {
+            "APIKEY": settings.securitytrails_api_key,
+            "Accept": "application/json"
+        }
+        url = f"https://api.securitytrails.com/v1/domain/{domain}/subdomains"
+        subs = []
+        
+        try:
+            resp = await self.http_client.get(
+                url=url,
+                headers=headers,
+                timeout=15.0,
+                max_retries=2,
+                raise_for_status=True
+            )
+            data = resp.json()
+            subdomains = data.get("subdomains", [])
+            for sub in subdomains:
+                subs.append(f"{sub}.{domain}")
+        except Exception as e:
+            logger.debug(f"Subdomain fetch error for {domain}: {e}")
+            
+        return subs
+
+    async def run(
+        self,
+        target: str,
+        context: Optional[Dict[str, Any]] = None,
+    ) -> List[Finding]:
+        """Execute SecurityTrails smart queries based on target input type."""
+        if not self.is_configured():
+            logger.warning("SecurityTrails API key is not configured. Skipping module.")
+            return []
+
+        target = target.strip()
+        context = context or {}
+        target_type = context.get("target_type", "domain")
+        findings: List[Finding] = []
+        
+        # ----------------------------------------------------
+        # Routing Logic based on Target Type
+        # ----------------------------------------------------
+        
+        if target_type == "ip":
+            self.notify(f"Querying SecurityTrails Reverse IP (DSL) for {target}...")
+            # Detect if it's a subnet (CIDR) or single IP
+            if "/" in target:
+                query = f"ipv4 = '{target}'"
+            else:
+                query = f"ipv4 = '{target}'"
+                
+            domains = await self._scroll_dsl_query(query)
+            if domains:
+                self.notify(f"Found {len(domains)} domains hosted on {target} via SecurityTrails.")
+                for d in domains:
+                    findings.append(Finding(
+                        type=FindingType.SUBDOMAIN if context.get("root_domain") and d.endswith(context.get("root_domain", "")) else FindingType.ASSOCIATED_DOMAIN,
+                        target=target,
+                        value=d,
+                        source="SecurityTrails",
+                        host_ip=target.split("/")[0],  # map back to target if it's an IP
+                        metadata={"description": f"SecurityTrails Reverse IP mapping to {target}"}
+                    ))
+
+        elif target_type == "email":
+            self.notify(f"Querying Reverse WHOIS for {target} via SecurityTrails...")
+            query = f"whois_email = '{target}'"
+            domains = await self._scroll_dsl_query(query)
+            if domains:
+                self.notify(f"Found {len(domains)} domains registered by {target}.")
+                for d in domains:
+                    findings.append(Finding(
+                        type=FindingType.ASSOCIATED_DOMAIN,
+                        target=target,
+                        value=d,
+                        source="SecurityTrails Reverse WHOIS",
+                        metadata={"description": f"Domain registered by {target}"}
+                    ))
+
+        elif target_type == "org":
+            self.notify(f"Querying Corporate Assets for {target} via SecurityTrails...")
+            query = f"whois_organization = '{target}' OR whois_name = '{target}'"
+            domains = await self._scroll_dsl_query(query)
+            if domains:
+                self.notify(f"Found {len(domains)} domains owned by organization {target}.")
+                for d in domains:
+                    findings.append(Finding(
+                        type=FindingType.ASSOCIATED_DOMAIN,
+                        target=target,
+                        value=d,
+                        source="SecurityTrails Org Recon",
+                        metadata={"description": f"Asset owned by {target}"}
+                    ))
+
+        elif target_type == "asn":
+            self.notify(f"Querying ASN Footprint for {target} via SecurityTrails...")
+            asn_num = target.replace("asn:", "").replace("AS", "").strip()
+            query = f"asn = '{asn_num}'"
+            domains = await self._scroll_dsl_query(query)
+            if domains:
+                self.notify(f"Found {len(domains)} domains hosted in AS{asn_num}.")
+                for d in domains:
+                    findings.append(Finding(
+                        type=FindingType.ASSOCIATED_DOMAIN,
+                        target=target,
+                        value=d,
+                        source="SecurityTrails ASN Mapping",
+                        metadata={"description": f"Domain hosted within AS{asn_num}"}
+                    ))
+
+        elif target_type in ("domain", "subdomain"):
+            self.notify(f"Running multi-layered SecurityTrails recon for {target}...")
+            
+            # Fire both Subdomain Enumeration and Historical DNS concurrently
+            subs_task = asyncio.create_task(self._fetch_subdomains(target))
+            hist_task = asyncio.create_task(self._fetch_historical_dns(target))
+            
+            subs, hist_ips = await asyncio.gather(subs_task, hist_task, return_exceptions=False)
+            
+            if subs:
+                self.notify(f"Found {len(subs)} subdomains for {target} via SecurityTrails.")
+                for sub in subs:
+                    findings.append(Finding(
+                        type=FindingType.SUBDOMAIN,
+                        target=target,
+                        value=sub,
+                        source="SecurityTrails",
+                        metadata={"description": f"SecurityTrails discovered subdomain {sub}"}
+                    ))
+            
+            if hist_ips:
+                self.notify(f"Found {len(hist_ips)} historical IPs for {target} (Anti-WAF).")
+                for ip in hist_ips:
+                    # Tag this specifically as a Historical IP
+                    finding = Finding(
+                        type=FindingType.HOST_INFO,
+                        target=target,
+                        value=ip,
+                        source="SecurityTrails Historical DNS",
+                        host_ip=ip,
+                        host_info=HostInfoData(ip=ip, hostnames=[target]),
+                        metadata={"description": f"Historical Origin IP mapped for {target}", "tags": ["Historical IP", "WAF Bypass Candidate"]}
+                    )
+                    findings.append(finding)
+
+        return findings