exploitgraph 1.0.0__py3-none-any.whl

modules/cloud/s3_enum.py

@@ -0,0 +1,469 @@
+"""
+ExploitGraph Module: S3 Bucket Enumerator
+Category: cloud
+Detects public S3 buckets via HTTP and optional boto3 API calls.
+Works WITHOUT AWS credentials (HTTP mode). Richer results WITH credentials (boto3 mode).
+
+Real-world AWS misconfigs detected:
+  - Public-read ACL (no credentials needed to list objects)
+  - Missing Block Public Access settings
+  - Bucket policy allowing s3:GetObject to *
+  - Directory listing enabled
+"""
+from __future__ import annotations
+import re
+import io
+import zipfile
+from typing import TYPE_CHECKING
+from pathlib import Path
+
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+from modules.base import BaseModule, ModuleResult
+
+if TYPE_CHECKING:
+    from core.session_manager import Session
+
+
+class S3Enum(BaseModule):
+
+    NAME        = "s3_enum"
+    DESCRIPTION = "Discover and audit public S3 buckets — HTTP-based + optional boto3 integration"
+    AUTHOR      = "ExploitGraph Team"
+    VERSION     = "1.2.0"
+    CATEGORY    = "cloud"
+    SEVERITY    = "CRITICAL"
+    MITRE       = ["T1530", "T1580"]
+    AWS_PARALLEL = "aws s3 ls s3://bucket --no-sign-request | aws s3api get-bucket-acl"
+
+    OPTIONS = {
+        "TARGET":        {"default": "",    "required": True,  "description": "Target base URL or S3 bucket URL"},
+        "BUCKET_NAME":   {"default": "",    "required": False, "description": "Specific bucket name to test"},
+        "WORDLIST":      {"default": "",    "required": False, "description": "Bucket name wordlist path"},
+        "AWS_PROFILE":   {"default": "",    "required": False, "description": "AWS CLI profile for boto3 mode"},
+        "AWS_REGION":    {"default": "us-east-1", "required": False, "description": "AWS region"},
+        "DOWNLOAD_FILES":{"default": "true","required": False, "description": "Download found files for analysis"},
+        "TIMEOUT":       {"default": "8",   "required": False, "description": "Request timeout"},
+    }
+
+    # S3 URL patterns for detecting bucket references in responses
+    _S3_PATTERNS = [
+        r's3\.amazonaws\.com/([a-z0-9][a-z0-9\-\.]{1,61}[a-z0-9])',
+        r'([a-z0-9][a-z0-9\-\.]{1,61}[a-z0-9])\.s3\.amazonaws\.com',
+        r'([a-z0-9][a-z0-9\-\.]{1,61}[a-z0-9])\.s3\.[a-z0-9-]+\.amazonaws\.com',
+        r's3://([a-z0-9][a-z0-9\-\.]{1,61}[a-z0-9])',
+    ]
+
+    def run(self, session: "Session") -> ModuleResult:
+        from core.config import cfg
+        from core.logger import log
+
+        target  = self.get_option("TARGET") or session.target
+        timeout = int(self.get_option("TIMEOUT", "8"))
+        download= self.get_option("DOWNLOAD_FILES", "true").lower() == "true"
+
+        self._timer_start()
+        log.section("S3 Bucket Enumeration & Audit")
+        log.info(f"MITRE: T1530 — Data from Cloud Storage Object")
+        log.info(f"AWS:   aws s3 ls s3://bucket --no-sign-request")
+
+        found_buckets = []
+        found_files   = []
+
+        # Step 1: Extract bucket names from target responses
+        bucket_names = self._discover_bucket_names(target, timeout, session)
+
+        # Step 2: Test each bucket for public access
+        for name in bucket_names:
+            result = self._test_bucket(name, timeout, download, session)
+            if result:
+                found_buckets.append(result)
+                found_files.extend(result.get("files", []))
+
+        # Step 3: Check target URL for exposed storage paths
+        self._check_storage_paths(target, timeout, download, session, found_files)
+
+        # Step 4: Optional boto3 mode
+        if self.get_option("AWS_PROFILE") or cfg.aws_enabled:
+            self._boto3_audit(bucket_names, session)
+
+        # Graph
+        if found_buckets or found_files:
+            session.add_graph_node("s3_exposure", f"Cloud Storage\nExposed ({len(found_files)} files)",
+                                   "exposure", "CRITICAL",
+                                   f"{len(found_buckets)} buckets, {len(found_files)} files")
+            session.add_graph_edge("http_enum", "s3_exposure", "finds exposed storage", "T1530")
+
+        elapsed = self._timer_stop()
+        log.success(f"S3 audit done in {elapsed}s — {len(found_buckets)} buckets, {len(found_files)} files")
+
+        return ModuleResult(True, {
+            "buckets_found": len(found_buckets),
+            "files_found":   len(found_files),
+            "buckets":       found_buckets,
+            "files":         found_files,
+        })
+
+    def _discover_bucket_names(self, target: str, timeout: int,
+                                session: "Session") -> list[str]:
+        from core.logger import log
+        names = set()
+
+        # User-specified bucket
+        if bucket := self.get_option("BUCKET_NAME"):
+            names.add(bucket)
+
+        # Extract from target hostname
+        # flaws.cloud → try "flaws.cloud" AND "flaws" as bucket names
+        from urllib.parse import urlparse
+        hostname = urlparse(target).hostname or ""
+        base = hostname.split(".")[0]
+
+        if hostname and hostname not in ("localhost", "127.0.0.1"):
+            names.add(hostname)           # e.g. "flaws.cloud" is itself a valid bucket name
+        if base and base not in ("localhost", "127", "www"):
+            names.add(base)               # e.g. "flaws"
+            for suffix in ["-backups", "-backup", "-assets", "-static", "-uploads",
+                           "-prod", "-staging", "-dev", "-data", "-logs", "-config"]:
+                names.add(f"{base}{suffix}")
+
+        # Scan target responses for S3 URLs
+        log.step("Scanning target responses for S3 bucket references...")
+        try:
+            r = requests.get(target, timeout=timeout, verify=False)
+            for pattern in self._S3_PATTERNS:
+                for match in re.findall(pattern, r.text, re.IGNORECASE):
+                    name = match.strip().lower()
+                    if 2 < len(name) < 64:
+                        names.add(name)
+                        log.found(f"S3 bucket reference in response: {name}")
+        except Exception:
+            pass
+
+        # Load wordlist
+        wl = self.get_option("WORDLIST", "")
+        if wl and Path(wl).exists():
+            for line in Path(wl).read_text().splitlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    names.add(line)
+                    if base:
+                        names.add(f"{base}-{line}")
+
+        return list(names)
+
+    def _test_bucket(self, name: str, timeout: int, download: bool,
+                      session: "Session") -> dict | None:
+        from core.logger import log
+        import xml.etree.ElementTree as ET
+
+        # All URL patterns to try — including anonymous S3 API (flaws.cloud pattern)
+        urls_to_try = [
+            f"https://{name}.s3.amazonaws.com/",
+            f"https://s3.amazonaws.com/{name}/",
+            f"https://{name}.s3.amazonaws.com/?list-type=2",
+            f"https://s3.amazonaws.com/{name}?list-type=2",
+            f"https://{name}.s3.us-east-1.amazonaws.com/",
+        ]
+
+        headers = {
+            "User-Agent": "ExploitGraph/1.0 (Security Research)",
+            # No Authorization = anonymous/unsigned request
+            # Equivalent to: aws s3 ls s3://bucket --no-sign-request
+        }
+
+        for url in urls_to_try:
+            try:
+                r = requests.get(url, headers=headers, timeout=timeout,
+                                 verify=False, allow_redirects=True)
+
+                if r.status_code == 200 and "ListBucketResult" in r.text:
+                    log.critical(f"PUBLIC S3 BUCKET: s3://{name}")
+                    log.info(f"AWS cmd: aws s3 ls s3://{name} --no-sign-request")
+                    log.info(f"AWS cmd: aws s3 sync s3://{name} ./ --no-sign-request")
+
+                    # Parse XML properly (handles namespaces)
+                    files = []
+                    try:
+                        xml_clean = re.sub(r' xmlns="[^"]+"', '', r.text)
+                        root = ET.fromstring(xml_clean)
+                        for content in root.findall('.//Contents'):
+                            key_el  = content.find('Key')
+                            size_el = content.find('Size')
+                            if key_el is not None:
+                                files.append(key_el.text)
+                        truncated = root.find('.//IsTruncated')
+                        if truncated is not None and truncated.text == 'true':
+                            log.warning("  Results truncated — use AWS CLI for full listing")
+                    except Exception:
+                        # Fallback regex
+                        files = re.findall(r"<Key>([^<]+)</Key>", r.text)
+
+                    log.success(f"  Objects in bucket: {len(files)}")
+                    for f in files[:10]:
+                        log.secret("Object", f)
+
+                    session.add_finding(
+                        module="s3_enum",
+                        title=f"Public S3 Bucket: {name}",
+                        severity="CRITICAL",
+                        description=(
+                            f"S3 bucket '{name}' allows anonymous listing. "
+                            f"{len(files)} objects are publicly accessible. "
+                            "This is equivalent to the flaws.cloud attack pattern."
+                        ),
+                        evidence=(
+                            f"URL: {url}\nHTTP 200 with XML listing\n"
+                            f"Objects ({len(files)}): {', '.join(files[:5])}"
+                            f"{'...' if len(files) > 5 else ''}\n"
+                            f"AWS: aws s3 ls s3://{name} --no-sign-request"
+                        ),
+                        recommendation=(
+                            "Enable S3 Block Public Access:\n"
+                            f"aws s3api put-public-access-block --bucket {name} "
+                            "--public-access-block-configuration "
+                            "BlockPublicAcls=true,IgnorePublicAcls=true,"
+                            "BlockPublicPolicy=true,RestrictPublicBuckets=true"
+                        ),
+                        cvss_score=9.8,
+                        aws_parallel="s3:ListBucket granted to AllUsers — flaws.cloud challenge 1 pattern",
+                        mitre_technique="T1530",
+                    )
+
+                    downloaded = []
+                    if download:
+                        downloaded = self._download_files(name, files, timeout, session)
+
+                    return {
+                        "name":    name,
+                        "url":     url,
+                        "objects": len(files),
+                        "files":   downloaded,
+                    }
+
+                elif r.status_code == 403:
+                    log.step(f"[403] s3://{name} — bucket exists but requires credentials")
+                elif r.status_code == 301:
+                    # Redirect to correct region — follow it
+                    location = r.headers.get("Location", "")
+                    if location and name in location:
+                        log.step(f"Redirecting to region endpoint: {location}")
+                        try:
+                            r2 = requests.get(location, headers=headers,
+                                              timeout=timeout, verify=False)
+                            if r2.status_code == 200 and "ListBucketResult" in r2.text:
+                                files = re.findall(r"<Key>([^<]+)</Key>", r2.text)
+                                log.critical(f"PUBLIC BUCKET (redirected region): s3://{name}")
+                                return {"name": name, "url": location,
+                                        "objects": len(files), "files": []}
+                        except Exception:
+                            pass
+            except Exception:
+                pass
+
+        return None
+
+    def _download_files(self, bucket: str, keys: list[str],
+                         timeout: int, session: "Session") -> list[dict]:
+        from core.logger import log
+        downloaded = []
+        INTERESTING = [".env", ".json", ".yaml", ".yml", ".conf", ".ini",
+                        ".key", ".pem", ".zip", ".tar", ".gz", ".sql",
+                        "config", "secret", "credential", "backup", "password"]
+
+        for key in keys[:20]:
+            if any(ext in key.lower() for ext in INTERESTING):
+                url = f"https://{bucket}.s3.amazonaws.com/{key}"
+                try:
+                    r = requests.get(url, timeout=timeout, verify=False)
+                    if r.status_code == 200:
+                        content = None
+                        log.found(f"Downloaded: {key} ({len(r.content)} bytes)")
+
+                        if key.endswith(".zip"):
+                            try:
+                                with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+                                    members = zf.namelist()
+                                    log.step(f"Archive members: {members}")
+                                    for member in members:
+                                        try:
+                                            text = zf.read(member).decode("utf-8", errors="ignore")
+                                            session.exposed_files.append({
+                                                "url":     f"{url}!/{member}",
+                                                "path":    f"zip:{member}",
+                                                "content": text,
+                                                "source":  "s3_enum",
+                                            })
+                                        except Exception:
+                                            pass
+                            except Exception:
+                                pass
+                        else:
+                            content = r.text
+                            session.exposed_files.append({
+                                "url":     url,
+                                "path":    key,
+                                "content": content,
+                                "source":  "s3_enum",
+                            })
+
+                        downloaded.append({"key": key, "url": url,
+                                            "size": len(r.content), "content": content})
+                except Exception:
+                    pass
+        return downloaded
+
+    def _check_storage_paths(self, target: str, timeout: int, download: bool,
+                               session: "Session", found_files: list):
+        """Check for storage-like paths on the target web server."""
+        from core.config import cfg
+        from core.logger import log
+
+        storage_paths = []
+        wl = cfg.wordlist_path("backup_files")
+        if wl.exists():
+            storage_paths = [l.strip() for l in wl.read_text().splitlines()
+                             if l.strip() and not l.startswith("#")]
+
+        if not storage_paths:
+            storage_paths = [
+                "/static/backups/", "/backups/", "/backup/", "/uploads/",
+                "/.env", "/.env.backup", "/.env.production",
+                "/config.json", "/config.yaml", "/app.yaml",
+            ]
+
+        for path in storage_paths:
+            url = f"{target}{path}"
+            try:
+                r = requests.get(url, timeout=timeout, verify=False)
+                if r.status_code == 200 and len(r.content) > 10:
+                    log.found(f"[{r.status_code}] Exposed: {url} ({len(r.content)} bytes)")
+                    file_info = {
+                        "url":     url,
+                        "path":    path,
+                        "content": r.text if not url.endswith(".zip") else None,
+                        "size":    len(r.content),
+                        "source":  "s3_enum",
+                    }
+                    session.exposed_files.append(file_info)
+                    found_files.append(file_info)
+
+                    # Handle ZIP archives
+                    if url.endswith(".zip") and download:
+                        try:
+                            with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+                                for member in zf.namelist():
+                                    text = zf.read(member).decode("utf-8", errors="ignore")
+                                    session.exposed_files.append({
+                                        "url": f"{url}!/{member}",
+                                        "path": f"zip:{member}",
+                                        "content": text,
+                                        "source": "s3_enum",
+                                    })
+                                    log.secret("Archive file", member)
+                        except Exception:
+                            pass
+
+                    session.add_finding(
+                        module="s3_enum",
+                        title=f"Exposed Storage File: {path}",
+                        severity="CRITICAL" if any(x in path for x in [".env","backup","config"]) else "HIGH",
+                        description=f"File accessible without authentication: {url}",
+                        evidence=f"HTTP 200 OK | Size: {len(r.content)} bytes",
+                        recommendation="Restrict access via authentication middleware. Never expose backup files publicly.",
+                        cvss_score=9.0,
+                        aws_parallel="S3 object with public-read ACL and no pre-signed URL requirement",
+                        mitre_technique="T1530",
+                    )
+            except Exception:
+                pass
+
+    def _boto3_audit(self, bucket_names: list[str], session: "Session"):
+        """Use boto3 for deeper bucket analysis (read-only operations only)."""
+        from core.logger import log
+        try:
+            import boto3
+            from botocore.exceptions import ClientError, NoCredentialsError
+
+            profile = self.get_option("AWS_PROFILE", "")
+            region  = self.get_option("AWS_REGION", "us-east-1")
+
+            try:
+                boto_session = boto3.Session(profile_name=profile or None, region_name=region)
+                s3 = boto_session.client("s3")
+
+                log.info("boto3 mode: running authenticated S3 audit...")
+
+                # List all accessible buckets
+                try:
+                    response = s3.list_buckets()
+                    all_buckets = [b["Name"] for b in response.get("Buckets", [])]
+                    log.found(f"boto3: Found {len(all_buckets)} accessible buckets")
+
+                    for bucket in all_buckets:
+                        self._boto3_check_bucket(s3, bucket, session)
+
+                except NoCredentialsError:
+                    log.warning("boto3: No AWS credentials found. Set AWS_PROFILE or configure ~/.aws/credentials")
+                except ClientError as e:
+                    log.warning(f"boto3: {e.response['Error']['Message']}")
+
+            except Exception as e:
+                log.warning(f"boto3 session error: {e}")
+
+        except ImportError:
+            log.step("boto3 not installed — HTTP-only mode. Install: pip install boto3")
+
+    def _boto3_check_bucket(self, s3, bucket_name: str, session: "Session"):
+        from core.logger import log
+        from botocore.exceptions import ClientError
+        try:
+            # Check Block Public Access
+            try:
+                bpa = s3.get_public_access_block(Bucket=bucket_name)
+                config = bpa["PublicAccessBlockConfiguration"]
+                if not all(config.values()):
+                    log.warning(f"[boto3] {bucket_name}: Block Public Access NOT fully enabled")
+                    session.add_finding(
+                        module="s3_enum",
+                        title=f"S3 Block Public Access Disabled: {bucket_name}",
+                        severity="HIGH",
+                        description="Block Public Access is not fully enabled on this S3 bucket.",
+                        evidence=f"Bucket: {bucket_name}\nConfig: {config}",
+                        recommendation="Run: aws s3api put-public-access-block --bucket {name} --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true",
+                        cvss_score=7.5,
+                        aws_parallel="s3:PutPublicAccessBlock remediation required",
+                        mitre_technique="T1530",
+                    )
+            except ClientError:
+                pass
+
+            # Check ACL
+            try:
+                acl = s3.get_bucket_acl(Bucket=bucket_name)
+                for grant in acl.get("Grants", []):
+                    grantee = grant.get("Grantee", {})
+                    if grantee.get("URI") in [
+                        "http://acs.amazonaws.com/groups/global/AllUsers",
+                        "http://acs.amazonaws.com/groups/global/AuthenticatedUsers",
+                    ]:
+                        perm = grant.get("Permission")
+                        log.critical(f"[boto3] {bucket_name}: Public ACL — {grantee['URI'].split('/')[-1]} has {perm}")
+                        session.add_finding(
+                            module="s3_enum",
+                            title=f"S3 Public ACL: {bucket_name}",
+                            severity="CRITICAL",
+                            description=f"Bucket {bucket_name} has a public ACL granting {perm} to {grantee['URI'].split('/')[-1]}",
+                            evidence=f"ACL Grant: {grant}",
+                            recommendation="Remove public ACL: aws s3api put-bucket-acl --bucket {name} --acl private",
+                            cvss_score=9.8,
+                            aws_parallel="s3:PutBucketAcl with 'private' ACL",
+                            mitre_technique="T1530",
+                        )
+            except ClientError:
+                pass
+
+        except Exception as e:
+            log.step(f"[boto3] Error checking {bucket_name}: {e}")