stryx-cli 0.1.0__py3-none-any.whl

stryx/crawler/graphql_discovery.py

@@ -0,0 +1,86 @@
+"""GraphQL endpoint discovery."""
+
+from __future__ import annotations
+
+import json
+
+from stryx.crawler.discovery import Endpoint
+from stryx.utils.http_client import HttpClient
+from stryx.utils.logging import get_logger
+
+logger = get_logger("crawler.graphql")
+
+GRAPHQL_PATHS = [
+    "/graphql",
+    "/graphiql",
+    "/api/graphql",
+    "/query",
+    "/api/query",
+    "/v1/graphql",
+    "/v2/graphql",
+    "/gql",
+    "/api/gql",
+    "/_graphql",
+]
+
+
+async def discover_graphql(target_url: str) -> list[Endpoint]:
+    """Discover GraphQL endpoints by probing common paths and introspection."""
+    logger.info("Scanning for GraphQL endpoints")
+    endpoints: list[Endpoint] = []
+    client = HttpClient(timeout=5)
+
+    introspection_query = json.dumps({
+        "query": "{ __schema { queryType { name } mutationType { name } types { name } } }"
+    })
+
+    for path in GRAPHQL_PATHS:
+        url = f"{target_url}{path}"
+        try:
+            # Try GET first (some servers support it)
+            response, evidence = await client.get(url)
+            if response.status_code == 200 and _looks_like_graphql(response.text):
+                endpoints.append(Endpoint(
+                    path=url,
+                    method="GET",
+                    source="graphql",
+                    confidence=0.9,
+                ))
+                continue
+
+            # Try POST with introspection query
+            response, evidence = await client.post(
+                url,
+                body=introspection_query,
+                headers={"Content-Type": "application/json"},
+            )
+            if response.status_code == 200:
+                try:
+                    data = response.json()
+                    if isinstance(data, dict) and "data" in data:
+                        logger.info(f"Found GraphQL endpoint at {path} (introspection works)")
+                        endpoints.append(Endpoint(
+                            path=url,
+                            method="POST",
+                            source="graphql-introspection",
+                            confidence=0.95,
+                        ))
+                except (json.JSONDecodeError, ValueError):
+                    pass
+        except Exception:
+            continue
+
+    return endpoints
+
+
+def _looks_like_graphql(text: str) -> bool:
+    """Check if a response looks like a GraphQL endpoint."""
+    indicators = [
+        "graphiql",
+        "graphql",
+        "__schema",
+        "query",
+        "mutation",
+    ]
+    text_lower = text.lower()
+    return any(ind in text_lower for ind in indicators)

stryx/crawler/html_crawler.py

@@ -0,0 +1,294 @@
+"""Recursive HTML web crawler with depth-limited link following.
+
+Follows <a href>, <form action>, <img src>, <script src>, <link href>,
+and <iframe src> to discover endpoints recursively.
+Respects crawl_depth and deduplicates URLs.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin, urlparse, urldefrag
+
+from stryx.crawler.discovery import Endpoint
+from stryx.utils.http_client import HttpClient
+from stryx.utils.logging import get_logger
+
+logger = get_logger("crawler.html")
+
+# Patterns to extract URLs from HTML
+_LINK_PATTERNS = [
+    # <a href="...">
+    re.compile(r'<a\s+[^>]*href=["\']([^"\']+)["\']', re.I),
+    # <form action="...">
+    re.compile(r'<form\s+[^>]*action=["\']([^"\']+)["\']', re.I),
+    # <img src="...">
+    re.compile(r'<img\s+[^>]*src=["\']([^"\']+)["\']', re.I),
+    # <script src="...">
+    re.compile(r'<script\s+[^>]*src=["\']([^"\']+)["\']', re.I),
+    # <link href="...">
+    re.compile(r'<link\s+[^>]*href=["\']([^"\']+)["\']', re.I),
+    # <iframe src="...">
+    re.compile(r'<iframe\s+[^>]*src=["\']([^"\']+)["\']', re.I),
+]
+
+# Patterns to extract form inputs
+_FORM_INPUT_PATTERN = re.compile(
+    r'<input\s+[^>]*name=["\']([^"\']+)["\']', re.I
+)
+_FORM_SELECT_PATTERN = re.compile(
+    r'<select\s+[^>]*name=["\']([^"\']+)["\']', re.I
+)
+_FORM_TEXTAREA_PATTERN = re.compile(
+    r'<textarea\s+[^>]*name=["\']([^"\']+)["\']', re.I
+)
+
+# Patterns to extract API-like paths from JavaScript
+_JS_API_PATTERNS = [
+    re.compile(r"""(?:fetch|axios|\.get|\.post|\.put|\.delete|\.patch)\s*\(\s*["']([^"']+)["']"""),
+    re.compile(r"""(?:url|endpoint|path|api)\s*[:=]\s*["']([^"']+)["']"""),
+    re.compile(r"""["'](/api/[^"']+)["']"""),
+    re.compile(r"""["'](/v[12]/[^"']+)["']"""),
+]
+
+# File extensions to skip (not HTML)
+_SKIP_EXTENSIONS = {
+    '.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
+    '.woff', '.woff2', '.ttf', '.eot', '.map', '.mp4', '.webm',
+    '.pdf', '.zip', '.tar', '.gz',
+}
+
+# Common ignored paths
+_IGNORED_PATHS = {
+    '/favicon.ico', '/robots.txt', '/sitemap.xml',
+    '/apple-touch-icon.png', '/manifest.json',
+}
+
+
+class HTMLCrawler:
+    """Recursive HTML crawler that follows links to discover endpoints."""
+
+    def __init__(
+        self,
+        target_url: str,
+        max_depth: int = 5,
+        max_pages: int = 500,
+        respect_robots: bool = False,
+    ):
+        self.target_url = target_url.rstrip("/")
+        self.base_domain = urlparse(self.target_url).netloc
+        self.max_depth = max_depth
+        self.max_pages = max_pages
+        self.respect_robots = respect_robots
+        self.client = HttpClient(timeout=10)
+        self.visited: set[str] = set()
+        self.endpoints: list[Endpoint] = []
+        self._robots_paths: set[str] = set()
+
+    async def crawl(self) -> list[Endpoint]:
+        """Start recursive crawling from the target URL."""
+        logger.info(f"Starting recursive crawl from {self.target_url} (depth={self.max_depth})")
+
+        if self.respect_robots:
+            await self._load_robots_txt()
+
+        await self._crawl_page(self.target_url, depth=0)
+
+        logger.info(f"Crawl complete: {len(self.endpoints)} endpoints discovered from {len(self.visited)} pages")
+        return self.endpoints
+
+    async def _crawl_page(self, url: str, depth: int) -> None:
+        """Crawl a single page and follow links."""
+        # Normalize URL
+        url = self._normalize_url(url)
+
+        # Check limits
+        if depth > self.max_depth:
+            return
+        if len(self.visited) >= self.max_pages:
+            return
+        if url in self.visited:
+            return
+
+        # Check if same domain
+        parsed = urlparse(url)
+        if parsed.netloc and parsed.netloc != self.base_domain:
+            return
+
+        # Check robots.txt
+        if self.respect_robots and self._is_disallowed(parsed.path):
+            return
+
+        # Skip non-HTML resources
+        path_lower = parsed.path.lower()
+        if any(path_lower.endswith(ext) for ext in _SKIP_EXTENSIONS):
+            return
+        if parsed.path in _IGNORED_PATHS:
+            return
+
+        self.visited.add(url)
+        logger.debug(f"Crawling [{depth}]: {url}")
+
+        try:
+            response, evidence = await self.client.get(url)
+
+            if response.status_code >= 400:
+                return
+
+            content_type = response.headers.get("content-type", "").lower()
+            if "text/html" not in content_type and "application/xhtml" not in content_type:
+                return
+
+            body = response.text
+
+            # Register this URL as an endpoint
+            self.endpoints.append(Endpoint(
+                path=url,
+                method="GET",
+                source="html_crawler",
+                confidence=0.9,
+            ))
+
+            # Extract and follow links
+            links = self._extract_links(body, url)
+            for link in links:
+                await self._crawl_page(link, depth + 1)
+
+            # Extract form actions and inputs
+            forms = self._extract_forms(body, url)
+            for form_url, params in forms:
+                self.endpoints.append(Endpoint(
+                    path=form_url,
+                    method="POST",
+                    source="html_form",
+                    confidence=0.85,
+                    params=params,
+                ))
+
+            # Extract API endpoints from inline JavaScript
+            js_endpoints = self._extract_js_endpoints(body, url)
+            for js_url in js_endpoints:
+                self.endpoints.append(Endpoint(
+                    path=js_url,
+                    method="GET",
+                    source="html_js_inline",
+                    confidence=0.7,
+                ))
+
+        except Exception as e:
+            logger.debug(f"Failed to crawl {url}: {e}")
+
+    def _extract_links(self, html: str, base_url: str) -> list[str]:
+        """Extract all links from HTML content."""
+        links = set()
+        for pattern in _LINK_PATTERNS:
+            for match in pattern.finditer(html):
+                href = match.group(1).strip()
+                if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+                    continue
+                full_url = self._resolve_url(href, base_url)
+                if full_url:
+                    links.add(full_url)
+        return list(links)
+
+    def _extract_forms(self, html: str, base_url: str) -> list[tuple[str, list[str]]]:
+        """Extract form actions and input parameter names."""
+        forms = []
+        # Find all <form> blocks
+        form_pattern = re.compile(r'<form\s+[^>]*action=["\']([^"\']+)["\'][^>]*>(.*?)</form>', re.I | re.S)
+        for match in form_pattern.finditer(html):
+            action = match.group(1).strip()
+            form_body = match.group(2)
+            full_url = self._resolve_url(action, base_url)
+            if not full_url:
+                continue
+
+            # Extract input names
+            params = []
+            for p in _FORM_INPUT_PATTERN.finditer(form_body):
+                params.append(p.group(1))
+            for p in _FORM_SELECT_PATTERN.finditer(form_body):
+                params.append(p.group(1))
+            for p in _FORM_TEXTAREA_PATTERN.finditer(form_body):
+                params.append(p.group(1))
+
+            forms.append((full_url, params))
+
+        return forms
+
+    def _extract_js_endpoints(self, html: str, base_url: str) -> list[str]:
+        """Extract API endpoints from inline JavaScript."""
+        endpoints = set()
+        # Find <script> blocks with inline code
+        script_pattern = re.compile(r'<script[^>]*>(.*?)</script>', re.I | re.S)
+        for match in script_pattern.finditer(html):
+            script_content = match.group(1)
+            if not script_content.strip():
+                continue
+            for pattern in _JS_API_PATTERNS:
+                for api_match in pattern.finditer(script_content):
+                    path = api_match.group(1)
+                    if path.startswith("/"):
+                        full_url = f"{self.base_domain}{path}"
+                        endpoints.add(full_url)
+        return list(endpoints)
+
+    def _resolve_url(self, href: str, base_url: str) -> str | None:
+        """Resolve a relative URL to absolute and normalize."""
+        try:
+            if href.startswith(("http://", "https://")):
+                full_url = href
+            else:
+                full_url = urljoin(base_url, href)
+
+            # Remove fragment
+            full_url, _ = urldefrag(full_url)
+
+            # Normalize
+            full_url = self._normalize_url(full_url)
+
+            return full_url
+        except Exception:
+            return None
+
+    def _normalize_url(self, url: str) -> str:
+        """Normalize a URL for deduplication."""
+        parsed = urlparse(url)
+        # Lowercase scheme and host
+        scheme = parsed.scheme.lower()
+        netloc = parsed.netloc.lower()
+        # Remove default ports
+        if netloc.endswith(":80") and scheme == "http":
+            netloc = netloc[:-3]
+        elif netloc.endswith(":443") and scheme == "https":
+            netloc = netloc[:-4]
+        # Normalize path
+        path = parsed.path.rstrip("/") or "/"
+        # Rebuild
+        normalized = f"{scheme}://{netloc}{path}"
+        if parsed.query:
+            normalized += f"?{parsed.query}"
+        return normalized
+
+    async def _load_robots_txt(self) -> None:
+        """Load and parse robots.txt Disallow rules."""
+        try:
+            robots_url = f"{self.base_domain}/robots.txt"
+            response, _ = await self.client.get(robots_url)
+            if response.status_code == 200:
+                for line in response.text.splitlines():
+                    line = line.strip()
+                    if line.lower().startswith("disallow:"):
+                        path = line.split(":", 1)[1].strip()
+                        if path:
+                            self._robots_paths.add(path)
+                logger.info(f"Loaded {len(self._robots_paths)} disallowed paths from robots.txt")
+        except Exception:
+            pass
+
+    def _is_disallowed(self, path: str) -> bool:
+        """Check if a path is disallowed by robots.txt."""
+        for disallowed in self._robots_paths:
+            if path.startswith(disallowed):
+                return True
+        return False

stryx/crawler/js_endpoints.py

@@ -0,0 +1,165 @@
+"""JavaScript endpoint extraction.
+
+Downloads and analyzes JavaScript files to discover API endpoints,
+HTTP methods, and URL patterns. Handles bundled/minified JS.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin, urlparse
+
+from stryx.crawler.discovery import Endpoint
+from stryx.utils.http_client import HttpClient
+from stryx.utils.logging import get_logger
+
+logger = get_logger("crawler.js_endpoints")
+
+# Patterns that commonly indicate API endpoints in JS
+ENDPOINT_PATTERNS = [
+    # fetch/axios/http client URLs
+    re.compile(r'''(?:fetch|axios|\.get|\.post|\.put|\.delete|\.patch|\.request)\s*\(\s*['"`]([^'"`]+)['"`]'''),
+    # URL assignments
+    re.compile(r'''(?:url|endpoint|api|path|href|baseUrl|baseURL)\s*[=:]\s*['"`]([^'"`]+)['"`]'''),
+    # Template literal API calls
+    re.compile(r'''`(/api/[^`]+)`'''),
+    re.compile(r'''`(/v\d/[^`]+)`'''),
+    re.compile(r'''`(/auth/[^`]+)`'''),
+    re.compile(r'''`(/admin/[^`]+)`'''),
+    # String concatenation API paths
+    re.compile(r'''['"]([^'"]*api[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/users\/[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/admin[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/login[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/auth[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/register[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/profile[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/settings[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/upload[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/search[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/graphql[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/websocket[^'"]*)['"]'''),
+    re.compile(r'''['"]([^'"]*\/ws[^'"]*)['"]'''),
+    # HTTP method detection
+    re.compile(r'''method\s*:\s*['"]?(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)['"]?'''),
+    # Common SPA routes
+    re.compile(r'''(?:path|route)\s*:\s*['"]\/([a-zA-Z][a-zA-Z0-9_\/-]*)['"]'''),
+]
+
+# HTTP method patterns (to detect which methods are used)
+METHOD_PATTERNS = {
+    "GET": re.compile(r'''\.get\s*\('''),
+    "POST": re.compile(r'''\.post\s*\('''),
+    "PUT": re.compile(r'''\.put\s*\('''),
+    "DELETE": re.compile(r'''\.delete\s*\('''),
+    "PATCH": re.compile(r'''\.patch\s*\('''),
+}
+
+# Patterns to skip (not endpoints)
+SKIP_PATTERNS = [
+    re.compile(r'''node_modules'''),
+    re.compile(r'''\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|map)$'''),
+    re.compile(r'''webpack'''),
+    re.compile(r'''bundle'''),
+    re.compile(r'''chunk'''),
+    re.compile(r'''favicon'''),
+    re.compile(r'''\.min\.''' ),
+    re.compile(r'''^(https?:)?//[a-z0-9.-]+\.(com|org|net|io)/$'''),  # Just a domain
+]
+
+
+async def discover_js_endpoints(target_url: str) -> list[Endpoint]:
+    """Discover endpoints embedded in JavaScript files."""
+    logger.info("Scanning JavaScript files for embedded endpoints")
+    endpoints: list[Endpoint] = []
+    client = HttpClient(timeout=10)
+    seen_urls: set[str] = set()
+
+    # First, try to find JS files from the page
+    try:
+        response, _ = await client.get(target_url)
+        if response.status_code == 200:
+            js_files = _extract_js_files(response.text, target_url)
+            logger.info(f"Found {len(js_files)} JS files to analyze")
+
+            for js_url in js_files[:20]:  # Limit to 20 files
+                try:
+                    js_response, _ = await client.get(js_url)
+                    if js_response.status_code == 200:
+                        found = _extract_endpoints_from_js(js_response.text, target_url)
+                        for ep in found:
+                            if ep.path not in seen_urls:
+                                seen_urls.add(ep.path)
+                                endpoints.append(ep)
+                except Exception:
+                    continue
+    except Exception as e:
+        logger.warning(f"Failed to scan JS files: {e}")
+
+    logger.info(f"Extracted {len(endpoints)} endpoints from JS files")
+    return endpoints
+
+
+def _extract_js_files(html: str, base_url: str) -> list[str]:
+    """Extract JavaScript file URLs from HTML."""
+    # Match script tags with src attribute
+    src_pattern = r'<script[^>]+src=["\']([^"\']+)["\']'
+    matches = re.findall(src_pattern, html, re.IGNORECASE)
+    return [urljoin(base_url, src) for src in matches]
+
+
+def _extract_endpoints_from_js(js_content: str, base_url: str) -> list[Endpoint]:
+    """Extract API endpoints from JavaScript source code."""
+    endpoints: list[Endpoint] = []
+    seen: set[str] = set()
+
+    # Detect HTTP methods used in this JS file
+    detected_methods: dict[str, str] = {}
+    for method, pattern in METHOD_PATTERNS.items():
+        if pattern.search(js_content):
+            detected_methods[method.lower()] = method
+
+    for pattern in ENDPOINT_PATTERNS:
+        matches = pattern.findall(js_content)
+        for match in matches:
+            # Skip very short or generic matches
+            if len(match) < 3 or match in seen:
+                continue
+            seen.add(match)
+
+            # Skip common non-endpoint strings
+            if any(re.search(skip, match, re.I) for skip in SKIP_PATTERNS):
+                continue
+
+            # Skip data URIs, blob URLs, etc.
+            if match.startswith(("data:", "blob:", "javascript:", "mailto:", "tel:")):
+                continue
+
+            # Determine if this is an absolute or relative URL
+            if match.startswith("http"):
+                url = match
+            elif match.startswith("/"):
+                url = f"{base_url.rstrip('/')}{match}"
+            else:
+                continue  # Skip relative paths without /
+
+            # Determine likely HTTP method from context
+            method = "GET"
+            # Look at surrounding context for method hints
+            for http_method, m_pattern in METHOD_PATTERNS.items():
+                # Check if this URL appears near a method call
+                context_start = max(0, js_content.find(match) - 100)
+                context_end = min(len(js_content), js_content.find(match) + len(match) + 100)
+                context = js_content[context_start:context_end]
+                if m_pattern.search(context):
+                    method = http_method
+                    break
+
+            endpoints.append(Endpoint(
+                path=url,
+                method=method,
+                source="js-endpoints",
+                confidence=0.65,
+            ))
+
+    return endpoints

stryx/crawler/openapi.py

@@ -0,0 +1,76 @@
+"""OpenAPI/Swagger endpoint discovery."""
+
+from __future__ import annotations
+
+from stryx.crawler.discovery import Endpoint
+from stryx.utils.http_client import HttpClient
+from stryx.utils.logging import get_logger
+
+logger = get_logger("crawler.openapi")
+
+OPENAPI_PATHS = [
+    "/openapi.json",
+    "/swagger.json",
+    "/api-docs",
+    "/swagger/v1/swagger.json",
+    "/v1/openapi.json",
+    "/v2/openapi.json",
+    "/v3/openapi.json",
+    "/api/openapi.json",
+    "/docs/openapi.json",
+    "/swagger.json",
+    "/swagger-ui.json",
+    "/api-docs.json",
+    "/api/swagger.json",
+    "/api/v1/swagger.json",
+    "/api/v2/swagger.json",
+]
+
+
+async def discover_openapi(target_url: str) -> list[Endpoint]:
+    """Discover endpoints from OpenAPI/Swagger documentation."""
+    logger.info("Scanning for OpenAPI/Swagger documentation")
+    endpoints: list[Endpoint] = []
+    client = HttpClient(timeout=5)
+
+    for path in OPENAPI_PATHS:
+        url = f"{target_url}{path}"
+        try:
+            response, evidence = await client.get(url)
+            if response.status_code == 200:
+                data = response.json()
+                if isinstance(data, dict) and ("paths" in data or "openapi" in data or "swagger" in data):
+                    logger.info(f"Found OpenAPI spec at {path}")
+                    endpoints.extend(_parse_openapi_paths(data, target_url))
+        except Exception:
+            continue
+
+    return endpoints
+
+
+def _parse_openapi_paths(spec: dict, base_url: str) -> list[Endpoint]:
+    """Parse paths from an OpenAPI specification."""
+    endpoints: list[Endpoint] = []
+    paths = spec.get("paths", {})
+
+    for path, methods in paths.items():
+        if not isinstance(methods, dict):
+            continue
+        for method in methods:
+            if method.lower() in ("get", "post", "put", "delete", "patch", "head", "options"):
+                params = []
+                operation = methods[method]
+                if isinstance(operation, dict):
+                    for param in operation.get("parameters", []):
+                        if isinstance(param, dict):
+                            params.append(param.get("name", ""))
+
+                endpoints.append(Endpoint(
+                    path=f"{base_url}{path}",
+                    method=method.upper(),
+                    source="openapi",
+                    confidence=0.95,
+                    params=[p for p in params if p],
+                ))
+
+    return endpoints