detecti-cli 2.0.0__py3-none-any.whl

detecti/config.py

@@ -0,0 +1,188 @@
+"""Configuration settings for DetecTI-CLI using Pydantic Settings."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+# Define the global base directory for DetecTI-CLI data
+custom_path = os.getenv("DETECTI_HOME", str(Path.home() / ".detecti"))
+DETECTI_HOME = Path(custom_path)
+DETECTI_HOME.mkdir(parents=True, exist_ok=True)
+
+
+def is_placeholder_key(val: Optional[str]) -> bool:
+    """Check if a string represents an unconfigured placeholder or template value."""
+    if not val or not isinstance(val, str):
+        return True
+    cleaned = val.strip().lower()
+    if not cleaned:
+        return True
+    if cleaned in ("none", "null", "undefined", "dummy", "xxx", "placeholder", "changeme", "example"):
+        return True
+    placeholder_prefixes_or_suffixes = (
+        "insert_your_",
+        "your_api",
+        "your_token",
+        "your_key",
+        "seu_",
+        "sua_",
+        "_aqui",
+        "_here",
+        "token_aqui",
+        "chave_aqui",
+        "api_key_here",
+        "changeme",
+        "<insert",
+    )
+    if any(p in cleaned for p in placeholder_prefixes_or_suffixes):
+        return True
+    if cleaned.startswith("<") and cleaned.endswith(">"):
+        return True
+    return False
+
+
+def sanitize_api_key(val: Optional[str]) -> Optional[str]:
+    """Return stripped string if valid and not a placeholder, else None."""
+    if not val or is_placeholder_key(val):
+        return None
+    return val.strip()
+
+
+def _find_legacy_api_key() -> Optional[str]:
+    """Look for legacy API.txt file in working directory or package root."""
+    candidate_paths = [
+        DETECTI_HOME / "API.txt",
+        Path.cwd() / "API.txt",
+        Path(__file__).resolve().parent / "API.txt",
+        Path(__file__).resolve().parent.parent / "API.txt",
+    ]
+    for path in candidate_paths:
+        if path.is_file():
+            try:
+                content = path.read_text(encoding="utf-8").strip()
+                sanitized = sanitize_api_key(content)
+                if sanitized:
+                    return sanitized
+            except Exception:
+                pass
+    return None
+
+
+class Settings(BaseSettings):
+    """DetecTI Application Settings."""
+
+    model_config = SettingsConfigDict(
+        env_file=(str(DETECTI_HOME / ".env"), ".env", "detecti-cli/.env", "threattrack/.env"),
+        env_file_encoding="utf-8",
+        extra="ignore",
+        env_prefix="DETECTI_",
+    )
+
+    # API Keys (can also be read from direct standard env vars like SHODAN_API_KEY)
+    shodan_api_key: Optional[str] = Field(
+        default=None,
+        validation_alias="SHODAN_API_KEY",
+        description="Shodan.io API key",
+    )
+    nvd_api_key: Optional[str] = Field(
+        default=None,
+        validation_alias="NVD_API_KEY",
+        description="National Vulnerability Database API key (optional, allows faster queries)",
+    )
+    whoisfreaks_api_key: Optional[str] = Field(
+        default=None,
+        validation_alias="WHOISFREAKS_API_KEY",
+        description="WhoisFreaks API key for reverse WHOIS",
+    )
+    github_token: Optional[str] = Field(
+        default=None,
+        validation_alias="GITHUB_TOKEN",
+        description="GitHub personal access token for PoC queries",
+    )
+    censys_pat_token: Optional[str] = Field(
+        default=None,
+        validation_alias="CENSYS_PAT_TOKEN",
+        description="Censys Platform API v3 Personal Access Token (PAT)",
+    )
+    censys_org_id: Optional[str] = Field(
+        default=None,
+        validation_alias="CENSYS_ORG_ID",
+        description="Censys Platform API v3 Organization ID (optional)",
+    )
+    censys_api_id: Optional[str] = Field(
+        default=None,
+        validation_alias="CENSYS_API_ID",
+        description="Legacy Censys Search API ID (fallback)",
+    )
+    censys_api_secret: Optional[str] = Field(
+        default=None,
+        validation_alias="CENSYS_API_SECRET",
+        description="Legacy Censys Search API Secret (fallback)",
+    )
+
+    # HTTP Client Configuration
+    http_timeout: float = Field(default=15.0, description="HTTP timeout in seconds")
+    http_max_retries: int = Field(default=3, description="Max HTTP retries for failed requests")
+    http_backoff_factor: float = Field(default=0.5, description="Exponential backoff factor")
+    http_concurrency_limit: int = Field(default=10, description="Max concurrent async requests")
+    user_agent: str = Field(
+        default="DetecTI-CLI/2.0 (+https://github.com/detectisec/DetecTI-CLI)",
+        description="HTTP User-Agent header",
+    )
+
+    # Rate Limiting Delays
+    nvd_delay_without_key: float = Field(default=6.0, description="Rate limit delay in seconds without NVD key")
+    nvd_delay_with_key: float = Field(default=0.6, description="Rate limit delay in seconds with NVD key")
+    hackertarget_delay: float = Field(default=1.0, description="Delay between HackerTarget free requests")
+    shodan_delay: float = Field(default=1.05, description="Rate limit delay in seconds for Shodan REST API (1 request/s limit)")
+
+    # Threat Intelligence Endpoint URLs
+    nvd_api_url: str = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+    epss_api_url: str = "https://api.first.org/data/v1/epss"
+    cisa_kev_url: str = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+    crtsh_api_url: str = "https://crt.sh"
+    github_poc_api_url: str = "https://poc-in-github.motikan2010.net/api/v1"
+    exploit_db_base_url: str = "https://www.exploit-db.com/exploits"
+    hackertarget_reverse_ip_url: str = "https://api.hackertarget.com/reverseiplookup"
+    hackertarget_whois_url: str = "https://api.hackertarget.com/whois"
+    whoisfreaks_reverse_whois_url: str = "https://api.whoisfreaks.com/v1.0/reversewhois"
+    censys_platform_api_url: str = "https://api.platform.censys.io/v3"
+    censys_hosts_api_url: str = "https://api.platform.censys.io/v3/global"
+
+    @model_validator(mode="after")
+    def populate_fallback_keys(self) -> Settings:
+        """Fallback to direct environment variables or API.txt if not set, filtering out placeholders."""
+        raw_shodan = self.shodan_api_key or os.getenv("SHODAN_API_KEY") or _find_legacy_api_key()
+        self.shodan_api_key = sanitize_api_key(raw_shodan)
+
+        raw_nvd = self.nvd_api_key or os.getenv("NVD_API_KEY")
+        self.nvd_api_key = sanitize_api_key(raw_nvd)
+
+        raw_whois = self.whoisfreaks_api_key or os.getenv("WHOISFREAKS_API_KEY")
+        self.whoisfreaks_api_key = sanitize_api_key(raw_whois)
+
+        raw_github = self.github_token or os.getenv("GITHUB_TOKEN")
+        self.github_token = sanitize_api_key(raw_github)
+
+        raw_censys_pat = self.censys_pat_token or os.getenv("CENSYS_PAT_TOKEN")
+        self.censys_pat_token = sanitize_api_key(raw_censys_pat)
+
+        raw_censys_org = self.censys_org_id or os.getenv("CENSYS_ORG_ID")
+        self.censys_org_id = sanitize_api_key(raw_censys_org)
+
+        raw_censys_id = self.censys_api_id or os.getenv("CENSYS_API_ID")
+        self.censys_api_id = sanitize_api_key(raw_censys_id)
+
+        raw_censys_secret = self.censys_api_secret or os.getenv("CENSYS_API_SECRET")
+        self.censys_api_secret = sanitize_api_key(raw_censys_secret)
+
+        return self
+
+
+# Global singleton settings instance
+settings = Settings()

detecti/core/__init__.py

@@ -0,0 +1 @@
+"""Core module package for ThreatTrack."""

detecti/core/database/__init__.py

@@ -0,0 +1,5 @@
+"""SQLite database layer for DetecTI-CLI EASM data persistence."""
+
+from .storage import DatabaseManager
+
+__all__ = ["DatabaseManager"]

detecti/core/database/config_db.py

@@ -0,0 +1,73 @@
+import logging
+import sqlite3
+import hashlib
+from pathlib import Path
+
+# Suppress passlib bcrypt warning
+try:
+    import bcrypt
+    if not hasattr(bcrypt, "__about__"):
+        class AboutMock:
+            __version__ = bcrypt.__version__
+        bcrypt.__about__ = AboutMock()
+except ImportError:
+    pass
+
+from passlib.context import CryptContext
+from pydantic_settings import BaseSettings
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+def get_password_hash(password: str) -> str:
+    return pwd_context.hash(password)
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    return pwd_context.verify(plain_password, hashed_password)
+
+class ConfigDBManager:
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.init_db()
+
+    def init_db(self):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute('''
+                CREATE TABLE IF NOT EXISTS users (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    username TEXT UNIQUE NOT NULL,
+                    password_hash TEXT NOT NULL,
+                    role TEXT DEFAULT 'admin',
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            ''')
+            conn.commit()
+
+    def create_user(self, username: str, password_hash: str, role: str = 'admin'):
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                conn.execute('''
+                    INSERT INTO users (username, password_hash, role)
+                    VALUES (?, ?, ?)
+                ''', (username, password_hash, role))
+                conn.commit()
+                return True
+        except sqlite3.IntegrityError:
+            return False
+
+    def update_user_password(self, username: str, password_hash: str):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute('''
+                UPDATE users SET password_hash = ? WHERE username = ?
+            ''', (password_hash, username))
+            conn.commit()
+
+    def get_user(self, username: str):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.execute("SELECT * FROM users WHERE username = ?", (username,))
+            return cursor.fetchone()
+
+    def user_exists(self, username: str) -> bool:
+        return self.get_user(username) is not None
+

detecti/core/database/schema.py

@@ -0,0 +1,136 @@
+"""SQLite schema definition for DetecTI-CLI EASM database."""
+
+SCHEMA_SQL = """
+-- Target Domains
+CREATE TABLE IF NOT EXISTS domains (
+    id TEXT PRIMARY KEY,
+    name TEXT UNIQUE NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Subdomains
+CREATE TABLE IF NOT EXISTS subdomains (
+    id TEXT PRIMARY KEY,
+    domain_id TEXT NOT NULL,
+    name TEXT NOT NULL,
+    status_code INTEGER,
+    cname TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (domain_id) REFERENCES domains(id)
+);
+
+-- IP Addresses
+CREATE TABLE IF NOT EXISTS ip_addresses (
+    id TEXT PRIMARY KEY,
+    ip TEXT UNIQUE NOT NULL,
+    asn TEXT,
+    org TEXT,
+    country TEXT,
+    city TEXT,
+    region_code TEXT,
+    postal_code TEXT,
+    latitude REAL,
+    longitude REAL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Mapping Subdomain to IP (DNS Resolutions)
+CREATE TABLE IF NOT EXISTS subdomain_ips (
+    subdomain_id TEXT NOT NULL,
+    ip_id TEXT NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (subdomain_id, ip_id),
+    FOREIGN KEY (subdomain_id) REFERENCES subdomains(id),
+    FOREIGN KEY (ip_id) REFERENCES ip_addresses(id)
+);
+
+-- Exposed Ports & Services
+CREATE TABLE IF NOT EXISTS services (
+    id TEXT PRIMARY KEY,
+    ip_id TEXT NOT NULL,
+    port INTEGER NOT NULL,
+    protocol TEXT DEFAULT 'tcp',
+    service_name TEXT,
+    product TEXT,
+    version TEXT,
+    banner TEXT,
+    url TEXT,
+    ssl BOOLEAN DEFAULT 0,
+    sources TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (ip_id) REFERENCES ip_addresses(id)
+);
+
+-- Vulnerabilities & EPSS/KEV Intelligence
+CREATE TABLE IF NOT EXISTS vulnerabilities (
+    id TEXT PRIMARY KEY,
+    service_id TEXT,
+    ip_id TEXT,
+    cve_id TEXT NOT NULL,
+    severity TEXT,
+    cvss_score REAL,
+    cvss_version TEXT,
+    description TEXT,
+    cwe_id TEXT,
+    cwe_name TEXT,
+    epss_score REAL,
+    epss_percentile REAL,
+    is_cisa_kev BOOLEAN DEFAULT 0,
+    cisa_kev_data TEXT,
+    source TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (service_id) REFERENCES services(id),
+    FOREIGN KEY (ip_id) REFERENCES ip_addresses(id)
+);
+
+-- Exploits and PoCs
+CREATE TABLE IF NOT EXISTS exploits (
+    id TEXT PRIMARY KEY,
+    vulnerability_id TEXT NOT NULL,
+    title TEXT NOT NULL,
+    source TEXT NOT NULL,
+    url TEXT NOT NULL,
+    verified BOOLEAN DEFAULT 0,
+    author TEXT,
+    date TEXT,
+    exploit_type TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (vulnerability_id) REFERENCES vulnerabilities(id)
+);
+
+-- Scan Results Metadata
+CREATE TABLE IF NOT EXISTS scan_results (
+    id TEXT PRIMARY KEY,
+    target TEXT NOT NULL,
+    target_type TEXT NOT NULL,
+    started_at TIMESTAMP NOT NULL,
+    completed_at TIMESTAMP,
+    elapsed_seconds REAL,
+    modules_run TEXT,
+    total_findings INTEGER DEFAULT 0,
+    total_hosts INTEGER DEFAULT 0,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Scan Live Logs Persistence
+CREATE TABLE IF NOT EXISTS scan_logs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    timestamp TEXT NOT NULL,
+    level TEXT NOT NULL,
+    message TEXT NOT NULL,
+    target TEXT,
+    input_target TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Create indexes for performance
+CREATE INDEX IF NOT EXISTS idx_subdomains_domain_id ON subdomains(domain_id);
+CREATE INDEX IF NOT EXISTS idx_services_ip_id ON services(ip_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerabilities_service_id ON vulnerabilities(service_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerabilities_ip_id ON vulnerabilities(ip_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerabilities_cve_id ON vulnerabilities(cve_id);
+CREATE INDEX IF NOT EXISTS idx_exploits_vulnerability_id ON exploits(vulnerability_id);
+CREATE INDEX IF NOT EXISTS idx_scan_results_target ON scan_results(target);
+CREATE INDEX IF NOT EXISTS idx_scan_logs_target ON scan_logs(target);
+CREATE INDEX IF NOT EXISTS idx_scan_logs_id ON scan_logs(id);
+"""