sophhub 0.4.19 → 0.4.21

package/skills/web-scraper/src/scripts/scrape.py

@@ -1,270 +1,270 @@
-#!/usr/bin/env python3
-"""
-Web scraper via headless Chrome print-to-pdf + PyMuPDF text extraction.
-No requests/beautifulsoup4 needed -- only uses packages already in the Docker image.
-Outputs JSON to stdout for agent consumption.
-"""
-
-import argparse
-import ipaddress
-import json
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-from typing import Dict, Optional
-from urllib.parse import urlparse
-
-# ---------------------------------------------------------------------------
-# SSRF guard
-# ---------------------------------------------------------------------------
-
-BLOCKED_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "[::1]", "metadata.google.internal"}
-
-
-def is_private_ip(hostname):
-    # type: (str) -> bool
-    try:
-        addr = ipaddress.ip_address(hostname)
-        return addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved
-    except ValueError:
-        return False
-
-
-def validate_url(url):
-    # type: (str) -> Optional[str]
-    parsed = urlparse(url)
-    if parsed.scheme not in ("http", "https"):
-        return "Unsupported scheme: %s. Only http/https allowed." % parsed.scheme
-    hostname = parsed.hostname or ""
-    if hostname in BLOCKED_HOSTS:
-        return "Blocked host: %s" % hostname
-    if is_private_ip(hostname):
-        return "Private/reserved IP blocked: %s" % hostname
-    return None
-
-
-# ---------------------------------------------------------------------------
-# Chrome binary detection
-# ---------------------------------------------------------------------------
-
-CHROME_CANDIDATES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
-
-
-def find_chrome():
-    # type: () -> Optional[str]
-    for name in CHROME_CANDIDATES:
-        path = shutil.which(name)
-        if path:
-            return path
-    return None
-
-
-# ---------------------------------------------------------------------------
-# Static scrape via curl (no extra deps)
-# ---------------------------------------------------------------------------
-
-def scrape_static(url, timeout):
-    # type: (str, int) -> Dict
-    """Lightweight fetch using curl + basic text extraction."""
-    curl = shutil.which("curl")
-    if not curl:
-        return {"status": "error", "mode": "static", "url": url,
-                "error": "curl not found on PATH"}
-
-    try:
-        result = subprocess.run(
-            [curl, "-sL", "--max-time", str(timeout),
-             "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/137.0",
-             "-H", "Accept: text/html,*/*;q=0.8",
-             url],
-            capture_output=True, text=True, timeout=timeout + 5,
-        )
-    except subprocess.TimeoutExpired:
-        return {"status": "error", "mode": "static", "url": url,
-                "error": "curl timed out after %ds" % timeout}
-
-    if result.returncode != 0:
-        return {"status": "error", "mode": "static", "url": url,
-                "error": "curl failed (code %d): %s" % (result.returncode, result.stderr[:300])}
-
-    html = result.stdout
-    text = _extract_text_from_html(html)
-    title = _extract_title_from_html(html)
-
-    return {
-        "status": "ok",
-        "mode": "static",
-        "url": url,
-        "title": title,
-        "text": text,
-        "length": len(text),
-    }
-
-
-def _extract_title_from_html(html):
-    # type: (str) -> str
-    """Extract <title> content without bs4."""
-    import re
-    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
-    return m.group(1).strip() if m else ""
-
-
-def _extract_text_from_html(html):
-    # type: (str) -> str
-    """Rough HTML-to-text without bs4: strip tags, collapse whitespace."""
-    import re
-    text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
-    text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
-    text = re.sub(r"<[^>]+>", "\n", text)
-    text = re.sub(r"&nbsp;", " ", text)
-    text = re.sub(r"&amp;", "&", text)
-    text = re.sub(r"&lt;", "<", text)
-    text = re.sub(r"&gt;", ">", text)
-    text = re.sub(r"&#\d+;", "", text)
-    text = re.sub(r"&\w+;", "", text)
-    lines = [line.strip() for line in text.split("\n") if line.strip()]
-    return "\n".join(lines)
-
-
-# ---------------------------------------------------------------------------
-# Dynamic scrape: Chrome --print-to-pdf + PyMuPDF
-# ---------------------------------------------------------------------------
-
-def scrape_dynamic(url, timeout):
-    # type: (str, int) -> Dict
-    chrome = find_chrome()
-    if not chrome:
-        return {
-            "status": "error", "mode": "dynamic", "url": url,
-            "error": "No Chrome/Chromium found. Searched: %s. Use --mode static instead."
-                     % ", ".join(CHROME_CANDIDATES),
-        }
-
-    tmpdir = tempfile.mkdtemp(prefix="web-scraper-")
-    pdf_path = os.path.join(tmpdir, "page.pdf")
-
-    try:
-        js_budget_ms = max(5000, timeout * 1000)
-        cmd = [
-            chrome,
-            "--headless=new",
-            "--print-to-pdf=" + pdf_path,
-            "--no-sandbox",
-            "--disable-gpu",
-            "--disable-software-rasterizer",
-            "--disable-dev-shm-usage",
-            "--no-pdf-header-footer",
-            "--virtual-time-budget=%d" % js_budget_ms,
-            url,
-        ]
-
-        try:
-            subprocess.run(
-                cmd, capture_output=True, text=True,
-                timeout=timeout + 30,
-            )
-        except subprocess.TimeoutExpired:
-            return {"status": "error", "mode": "dynamic", "url": url,
-                    "error": "Chrome timed out after %ds" % timeout}
-
-        if not os.path.exists(pdf_path) or os.path.getsize(pdf_path) == 0:
-            return {"status": "error", "mode": "dynamic", "url": url,
-                    "error": "Chrome did not produce a PDF file"}
-
-        text = _extract_text_from_pdf(pdf_path)
-        title = _guess_title(text)
-
-        return {
-            "status": "ok",
-            "mode": "dynamic",
-            "url": url,
-            "title": title,
-            "text": text,
-            "length": len(text),
-        }
-    finally:
-        try:
-            shutil.rmtree(tmpdir, ignore_errors=True)
-        except Exception:
-            pass
-
-
-def _extract_text_from_pdf(pdf_path):
-    # type: (str) -> str
-    """Extract text from PDF using PyMuPDF (already in requirements.txt)."""
-    try:
-        import fitz  # PyMuPDF
-    except ImportError:
-        return _extract_text_from_pdf_fallback(pdf_path)
-
-    doc = fitz.open(pdf_path)
-    pages = []
-    for page in doc:
-        pages.append(page.get_text())
-    doc.close()
-    text = "\n".join(pages).strip()
-    lines = [line.strip() for line in text.split("\n") if line.strip()]
-    return "\n".join(lines)
-
-
-def _extract_text_from_pdf_fallback(pdf_path):
-    # type: (str) -> str
-    """Fallback: try pdfminer if PyMuPDF is not available."""
-    try:
-        from pdfminer.high_level import extract_text
-        return extract_text(pdf_path).strip()
-    except ImportError:
-        return "[error] Neither PyMuPDF nor pdfminer available. Install with: pip3 install --break-system-packages PyMuPDF"
-
-
-def _guess_title(text):
-    # type: (str) -> str
-    """Use the first non-empty line as title."""
-    for line in text.split("\n"):
-        stripped = line.strip()
-        if stripped and len(stripped) < 200:
-            return stripped
-    return ""
-
-
-# ---------------------------------------------------------------------------
-# Main
-# ---------------------------------------------------------------------------
-
-def main():
-    parser = argparse.ArgumentParser(description="Web scraper for OpenClaw agents")
-    parser.add_argument("url", help="URL to scrape (http/https only)")
-    parser.add_argument("--mode", choices=["static", "dynamic"], default="static",
-                        help="static: curl + text extract; dynamic: Chrome PDF + PyMuPDF (default: static)")
-    parser.add_argument("--timeout", type=int, default=15,
-                        help="Request timeout in seconds (default: 15)")
-    parser.add_argument("--max-chars", type=int, default=50000,
-                        help="Max output characters (default: 50000)")
-    args = parser.parse_args()
-
-    err = validate_url(args.url)
-    if err:
-        print(json.dumps({"status": "error", "url": args.url, "error": err}, ensure_ascii=False))
-        sys.exit(1)
-
-    try:
-        if args.mode == "static":
-            result = scrape_static(args.url, args.timeout)
-        else:
-            result = scrape_dynamic(args.url, args.timeout)
-    except Exception as e:
-        result = {"status": "error", "url": args.url, "mode": args.mode,
-                  "error": "%s: %s" % (type(e).__name__, e)}
-
-    if result.get("status") == "ok" and len(result.get("text", "")) > args.max_chars:
-        result["text"] = result["text"][:args.max_chars] + "\n\n... [truncated]"
-        result["truncated"] = True
-        result["length"] = len(result["text"])
-
-    print(json.dumps(result, ensure_ascii=False, indent=2))
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""
+Web scraper via headless Chrome print-to-pdf + PyMuPDF text extraction.
+No requests/beautifulsoup4 needed -- only uses packages already in the Docker image.
+Outputs JSON to stdout for agent consumption.
+"""
+
+import argparse
+import ipaddress
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Dict, Optional
+from urllib.parse import urlparse
+
+# ---------------------------------------------------------------------------
+# SSRF guard
+# ---------------------------------------------------------------------------
+
+BLOCKED_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "[::1]", "metadata.google.internal"}
+
+
+def is_private_ip(hostname):
+    # type: (str) -> bool
+    try:
+        addr = ipaddress.ip_address(hostname)
+        return addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved
+    except ValueError:
+        return False
+
+
+def validate_url(url):
+    # type: (str) -> Optional[str]
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        return "Unsupported scheme: %s. Only http/https allowed." % parsed.scheme
+    hostname = parsed.hostname or ""
+    if hostname in BLOCKED_HOSTS:
+        return "Blocked host: %s" % hostname
+    if is_private_ip(hostname):
+        return "Private/reserved IP blocked: %s" % hostname
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Chrome binary detection
+# ---------------------------------------------------------------------------
+
+CHROME_CANDIDATES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
+
+
+def find_chrome():
+    # type: () -> Optional[str]
+    for name in CHROME_CANDIDATES:
+        path = shutil.which(name)
+        if path:
+            return path
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Static scrape via curl (no extra deps)
+# ---------------------------------------------------------------------------
+
+def scrape_static(url, timeout):
+    # type: (str, int) -> Dict
+    """Lightweight fetch using curl + basic text extraction."""
+    curl = shutil.which("curl")
+    if not curl:
+        return {"status": "error", "mode": "static", "url": url,
+                "error": "curl not found on PATH"}
+
+    try:
+        result = subprocess.run(
+            [curl, "-sL", "--max-time", str(timeout),
+             "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/137.0",
+             "-H", "Accept: text/html,*/*;q=0.8",
+             url],
+            capture_output=True, text=True, timeout=timeout + 5,
+        )
+    except subprocess.TimeoutExpired:
+        return {"status": "error", "mode": "static", "url": url,
+                "error": "curl timed out after %ds" % timeout}
+
+    if result.returncode != 0:
+        return {"status": "error", "mode": "static", "url": url,
+                "error": "curl failed (code %d): %s" % (result.returncode, result.stderr[:300])}
+
+    html = result.stdout
+    text = _extract_text_from_html(html)
+    title = _extract_title_from_html(html)
+
+    return {
+        "status": "ok",
+        "mode": "static",
+        "url": url,
+        "title": title,
+        "text": text,
+        "length": len(text),
+    }
+
+
+def _extract_title_from_html(html):
+    # type: (str) -> str
+    """Extract <title> content without bs4."""
+    import re
+    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    return m.group(1).strip() if m else ""
+
+
+def _extract_text_from_html(html):
+    # type: (str) -> str
+    """Rough HTML-to-text without bs4: strip tags, collapse whitespace."""
+    import re
+    text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
+    text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
+    text = re.sub(r"<[^>]+>", "\n", text)
+    text = re.sub(r"&nbsp;", " ", text)
+    text = re.sub(r"&amp;", "&", text)
+    text = re.sub(r"&lt;", "<", text)
+    text = re.sub(r"&gt;", ">", text)
+    text = re.sub(r"&#\d+;", "", text)
+    text = re.sub(r"&\w+;", "", text)
+    lines = [line.strip() for line in text.split("\n") if line.strip()]
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Dynamic scrape: Chrome --print-to-pdf + PyMuPDF
+# ---------------------------------------------------------------------------
+
+def scrape_dynamic(url, timeout):
+    # type: (str, int) -> Dict
+    chrome = find_chrome()
+    if not chrome:
+        return {
+            "status": "error", "mode": "dynamic", "url": url,
+            "error": "No Chrome/Chromium found. Searched: %s. Use --mode static instead."
+                     % ", ".join(CHROME_CANDIDATES),
+        }
+
+    tmpdir = tempfile.mkdtemp(prefix="web-scraper-")
+    pdf_path = os.path.join(tmpdir, "page.pdf")
+
+    try:
+        js_budget_ms = max(5000, timeout * 1000)
+        cmd = [
+            chrome,
+            "--headless=new",
+            "--print-to-pdf=" + pdf_path,
+            "--no-sandbox",
+            "--disable-gpu",
+            "--disable-software-rasterizer",
+            "--disable-dev-shm-usage",
+            "--no-pdf-header-footer",
+            "--virtual-time-budget=%d" % js_budget_ms,
+            url,
+        ]
+
+        try:
+            subprocess.run(
+                cmd, capture_output=True, text=True,
+                timeout=timeout + 30,
+            )
+        except subprocess.TimeoutExpired:
+            return {"status": "error", "mode": "dynamic", "url": url,
+                    "error": "Chrome timed out after %ds" % timeout}
+
+        if not os.path.exists(pdf_path) or os.path.getsize(pdf_path) == 0:
+            return {"status": "error", "mode": "dynamic", "url": url,
+                    "error": "Chrome did not produce a PDF file"}
+
+        text = _extract_text_from_pdf(pdf_path)
+        title = _guess_title(text)
+
+        return {
+            "status": "ok",
+            "mode": "dynamic",
+            "url": url,
+            "title": title,
+            "text": text,
+            "length": len(text),
+        }
+    finally:
+        try:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+        except Exception:
+            pass
+
+
+def _extract_text_from_pdf(pdf_path):
+    # type: (str) -> str
+    """Extract text from PDF using PyMuPDF (already in requirements.txt)."""
+    try:
+        import fitz  # PyMuPDF
+    except ImportError:
+        return _extract_text_from_pdf_fallback(pdf_path)
+
+    doc = fitz.open(pdf_path)
+    pages = []
+    for page in doc:
+        pages.append(page.get_text())
+    doc.close()
+    text = "\n".join(pages).strip()
+    lines = [line.strip() for line in text.split("\n") if line.strip()]
+    return "\n".join(lines)
+
+
+def _extract_text_from_pdf_fallback(pdf_path):
+    # type: (str) -> str
+    """Fallback: try pdfminer if PyMuPDF is not available."""
+    try:
+        from pdfminer.high_level import extract_text
+        return extract_text(pdf_path).strip()
+    except ImportError:
+        return "[error] Neither PyMuPDF nor pdfminer available. Install with: pip3 install --break-system-packages PyMuPDF"
+
+
+def _guess_title(text):
+    # type: (str) -> str
+    """Use the first non-empty line as title."""
+    for line in text.split("\n"):
+        stripped = line.strip()
+        if stripped and len(stripped) < 200:
+            return stripped
+    return ""
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="Web scraper for OpenClaw agents")
+    parser.add_argument("url", help="URL to scrape (http/https only)")
+    parser.add_argument("--mode", choices=["static", "dynamic"], default="static",
+                        help="static: curl + text extract; dynamic: Chrome PDF + PyMuPDF (default: static)")
+    parser.add_argument("--timeout", type=int, default=15,
+                        help="Request timeout in seconds (default: 15)")
+    parser.add_argument("--max-chars", type=int, default=50000,
+                        help="Max output characters (default: 50000)")
+    args = parser.parse_args()
+
+    err = validate_url(args.url)
+    if err:
+        print(json.dumps({"status": "error", "url": args.url, "error": err}, ensure_ascii=False))
+        sys.exit(1)
+
+    try:
+        if args.mode == "static":
+            result = scrape_static(args.url, args.timeout)
+        else:
+            result = scrape_dynamic(args.url, args.timeout)
+    except Exception as e:
+        result = {"status": "error", "url": args.url, "mode": args.mode,
+                  "error": "%s: %s" % (type(e).__name__, e)}
+
+    if result.get("status") == "ok" and len(result.get("text", "")) > args.max_chars:
+        result["text"] = result["text"][:args.max_chars] + "\n\n... [truncated]"
+        result["truncated"] = True
+        result["length"] = len(result["text"])
+
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/skills/website-builder/skill.json

@@ -1,20 +1,20 @@
-{
-  "name": "website-builder",
-  "version": "1.0.0",
-  "types": [
-    "builtin"
-  ],
-  "displayName": "",
-  "description": "",
-  "changelog": [
-    {
-      "version": "1.0.0",
-      "date": "2026-04-09",
-      "changes": [
-        "初次提交"
-      ]
-    }
-  ],
-  "createdAt": "2026-04-09",
-  "updatedAt": "2026-04-09"
-}
+{
+  "name": "website-builder",
+  "version": "1.0.0",
+  "types": [
+    "builtin"
+  ],
+  "displayName": "",
+  "description": "",
+  "changelog": [
+    {
+      "version": "1.0.0",
+      "date": "2026-04-09",
+      "changes": [
+        "初次提交"
+      ]
+    }
+  ],
+  "createdAt": "2026-04-09",
+  "updatedAt": "2026-04-09"
+}