sibylline-scurl 0.1.1__tar.gz → 0.2.3__tar.gz

{sibylline_scurl-0.1.1 → sibylline_scurl-0.2.3}/.github/workflows/ci.yml

@@ -3,6 +3,7 @@ name: CI
 on:
   push:
     branches: [main]
+    tags: ['v*']
   pull_request:
     branches: [main]
 
@@ -24,7 +25,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev]" pytest-mock
+          pip install -e ".[dev,prompt-defender]" pytest-mock
 
       - name: Run tests
         run: pytest -v
@@ -44,3 +45,26 @@ jobs:
 
       - name: Run ruff
         run: ruff check src/ tests/
+
+  publish:
+    needs: [test, lint]
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

sibylline_scurl-0.2.3/.github/workflows/docs.yml

@@ -0,0 +1,60 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'site/**'
+      - 'docs/**'
+      - '.github/workflows/docs.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: site/package-lock.json
+
+      - name: Install dependencies
+        working-directory: site
+        run: npm ci
+
+      - name: Build site
+        working-directory: site
+        run: npm run build:production
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site/_site
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

{sibylline_scurl-0.1.1 → sibylline_scurl-0.2.3}/.gitignore

@@ -41,3 +41,10 @@ htmlcov/
 # OS
 .DS_Store
 Thumbs.db
+
+# Generated docs site
+site/node_modules/
+site/_site/
+
+# Arbiter (project management)
+.arbiter/

sibylline_scurl-0.2.3/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: sibylline-scurl
+Version: 0.2.3
+Summary: A secure curl wrapper with middleware support and HTML-to-markdown extraction
+Author: Nathan
+License: MIT
+Keywords: curl,markdown,security,web-scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Security
+Requires-Python: >=3.10
+Requires-Dist: html2text>=2024.2.26
+Requires-Dist: readability-lxml>=0.8.1
+Provides-Extra: browser
+Requires-Dist: playwright>=1.40.0; extra == 'browser'
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Provides-Extra: prompt-defender
+Requires-Dist: confusable-homoglyphs>=3.2.0; extra == 'prompt-defender'
+Requires-Dist: huggingface-hub>=0.20.0; extra == 'prompt-defender'
+Requires-Dist: numpy>=1.24.0; extra == 'prompt-defender'
+Requires-Dist: onnxruntime>=1.17.0; extra == 'prompt-defender'
+Requires-Dist: rapidfuzz>=3.5.0; extra == 'prompt-defender'
+Requires-Dist: scikit-learn>=1.3.0; extra == 'prompt-defender'
+Requires-Dist: tokenizers>=0.15.0; extra == 'prompt-defender'
+Provides-Extra: training
+Requires-Dist: confusable-homoglyphs>=3.2.0; extra == 'training'
+Requires-Dist: datasets>=2.14.0; extra == 'training'
+Requires-Dist: huggingface-hub>=0.20.0; extra == 'training'
+Requires-Dist: numpy>=1.24.0; extra == 'training'
+Requires-Dist: onnxruntime>=1.17.0; extra == 'training'
+Requires-Dist: scikit-learn>=1.3.0; extra == 'training'
+Requires-Dist: tokenizers>=0.15.0; extra == 'training'
+Description-Content-Type: text/markdown
+
+# scurl
+
+[![PyPI version](https://badge.fury.io/py/scurl.svg)](https://badge.fury.io/py/scurl)
+[![CI](https://github.com/yourusername/scurl/actions/workflows/ci.yml/badge.svg)](https://github.com/yourusername/scurl/actions/workflows/ci.yml)
+
+A secure curl wrapper with middleware support and HTML-to-markdown extraction.
+
+## Installation
+
+```bash
+pip install sibylline-scurl
+```
+
+Or with [pipx](https://pipx.pypa.io/) (recommended for CLI tools):
+
+```bash
+pipx install sibylline-scurl
+```
+
+## Usage
+
+```bash
+# Fetch a URL and extract clean markdown from HTML
+scurl https://example.com
+
+# Raw output (disable response middleware)
+scurl --raw https://example.com
+
+# All curl flags work
+scurl -H "Accept: application/json" https://api.example.com/data
+```
+
+## Features
+
+- **SecretDefender**: Automatically detects and blocks requests containing exposed secrets/tokens
+- **HTML to Markdown**: Converts HTML responses to clean markdown (use `--readability` for article extraction)
+- **Middleware System**: Composable request and response middleware
+
+## Why scurl?
+
+scurl extracts clean, readable content from web pages - perfect for LLM consumption, readability, or bandwidth savings.
+
+### Size Comparison
+
+| Website | curl | scurl | Reduction |
+|---------|------|-------|-----------|
+| example.com | 513 | 167 | 67.4% |
+| news.ycombinator.com | 34,082 | 10,739 | 68.5% |
+| en.wikipedia.org/wiki/Curl | 110,373 | 10,044 | 90.9% |
+| github.com/anthropics | 296,788 | 353 | 99.9% |
+| docs.python.org | 319,554 | 12,348 | 96.1% |
+
+### Visual Comparison
+
+**curl output** (Wikipedia, first 500 chars):
+```html
+<!DOCTYPE html><html class="client-nojs" lang="en" dir="ltr"><head>
+<meta charset="UTF-8"/><title>Curl (programming language) - Wikipedia</title>
+<script>(function(){var className="client-js";var cookie=document.cookie.
+match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].
+split('%2C').forEach(function(pref){className=className.replace(new
+RegExp('(^| )'+pref.replace(/-hierarchical-hierarchical/,'')
++'($| )'),'$1teleported-hierarchical$2');});...
+```
+
+**scurl output** (same page):
+```markdown
+# Curl (programming language) - Wikipedia
+
+**Curl** is a reflective object-oriented programming language for interactive
+web applications, whose goal is to provide a smoother transition between
+content formatting and computer programming. It makes it possible to embed
+complex objects in simple documents without needing to switch between
+programming languages or development platforms.
+
+The Curl implementation initially consisted of an interpreter only; a compiler
+was added later...
+```
+
+## Flags
+
+| Flag | Description |
+|------|-------------|
+| `--raw` | Disable all response middleware (raw HTML output) |
+| `--readability` | Extract article content only (strips nav, ads, sidebars) |
+| `--render` | Use headless browser for JS-rendered pages |
+| `--disable <slug>` | Disable a middleware by slug (can be repeated) |
+| `--enable <slug>` | Override a middleware's block (can be repeated) |
+| `--list-middleware` | List available middleware and their slugs |
+
+## Middleware Slugs
+
+| Slug | Type | Description |
+|------|------|-------------|
+| `secret-defender` | Request | Detects and blocks requests containing secrets |
+| `readability` | Response | Extracts clean markdown from HTML |
+
+## License
+
+Copyright 2026 [Sibylline Software](https://sibylline.dev)
+
+MIT

sibylline_scurl-0.2.3/README.md

@@ -0,0 +1,102 @@
+# scurl
+
+[![PyPI version](https://badge.fury.io/py/scurl.svg)](https://badge.fury.io/py/scurl)
+[![CI](https://github.com/yourusername/scurl/actions/workflows/ci.yml/badge.svg)](https://github.com/yourusername/scurl/actions/workflows/ci.yml)
+
+A secure curl wrapper with middleware support and HTML-to-markdown extraction.
+
+## Installation
+
+```bash
+pip install sibylline-scurl
+```
+
+Or with [pipx](https://pipx.pypa.io/) (recommended for CLI tools):
+
+```bash
+pipx install sibylline-scurl
+```
+
+## Usage
+
+```bash
+# Fetch a URL and extract clean markdown from HTML
+scurl https://example.com
+
+# Raw output (disable response middleware)
+scurl --raw https://example.com
+
+# All curl flags work
+scurl -H "Accept: application/json" https://api.example.com/data
+```
+
+## Features
+
+- **SecretDefender**: Automatically detects and blocks requests containing exposed secrets/tokens
+- **HTML to Markdown**: Converts HTML responses to clean markdown (use `--readability` for article extraction)
+- **Middleware System**: Composable request and response middleware
+
+## Why scurl?
+
+scurl extracts clean, readable content from web pages - perfect for LLM consumption, readability, or bandwidth savings.
+
+### Size Comparison
+
+| Website | curl | scurl | Reduction |
+|---------|------|-------|-----------|
+| example.com | 513 | 167 | 67.4% |
+| news.ycombinator.com | 34,082 | 10,739 | 68.5% |
+| en.wikipedia.org/wiki/Curl | 110,373 | 10,044 | 90.9% |
+| github.com/anthropics | 296,788 | 353 | 99.9% |
+| docs.python.org | 319,554 | 12,348 | 96.1% |
+
+### Visual Comparison
+
+**curl output** (Wikipedia, first 500 chars):
+```html
+<!DOCTYPE html><html class="client-nojs" lang="en" dir="ltr"><head>
+<meta charset="UTF-8"/><title>Curl (programming language) - Wikipedia</title>
+<script>(function(){var className="client-js";var cookie=document.cookie.
+match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].
+split('%2C').forEach(function(pref){className=className.replace(new
+RegExp('(^| )'+pref.replace(/-hierarchical-hierarchical/,'')
++'($| )'),'$1teleported-hierarchical$2');});...
+```
+
+**scurl output** (same page):
+```markdown
+# Curl (programming language) - Wikipedia
+
+**Curl** is a reflective object-oriented programming language for interactive
+web applications, whose goal is to provide a smoother transition between
+content formatting and computer programming. It makes it possible to embed
+complex objects in simple documents without needing to switch between
+programming languages or development platforms.
+
+The Curl implementation initially consisted of an interpreter only; a compiler
+was added later...
+```
+
+## Flags
+
+| Flag | Description |
+|------|-------------|
+| `--raw` | Disable all response middleware (raw HTML output) |
+| `--readability` | Extract article content only (strips nav, ads, sidebars) |
+| `--render` | Use headless browser for JS-rendered pages |
+| `--disable <slug>` | Disable a middleware by slug (can be repeated) |
+| `--enable <slug>` | Override a middleware's block (can be repeated) |
+| `--list-middleware` | List available middleware and their slugs |
+
+## Middleware Slugs
+
+| Slug | Type | Description |
+|------|------|-------------|
+| `secret-defender` | Request | Detects and blocks requests containing secrets |
+| `readability` | Response | Extracts clean markdown from HTML |
+
+## License
+
+Copyright 2026 [Sibylline Software](https://sibylline.dev)
+
+MIT

sibylline_scurl-0.2.3/cf/.gitignore

@@ -0,0 +1,9 @@
+# Dependencies
+node_modules/
+
+# Wrangler
+.wrangler/
+.dev.vars
+
+# Build artifacts
+*.log

sibylline_scurl-0.2.3/cf/container/.gitignore

@@ -0,0 +1,2 @@
+# Build artifacts - wheel is copied in before docker build
+*.whl

sibylline_scurl-0.2.3/cf/container/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install scurl from local wheel with browser extra
+COPY *.whl /tmp/
+RUN pip install --no-cache-dir "/tmp/sibylline_scurl-0.2.0-py3-none-any.whl[browser,prompt-defender]" numpy && \
+    playwright install chromium --only-shell && \
+    playwright install-deps chromium && \
+    rm -f /tmp/*.whl
+
+# Pre-download MiniLM model (86MB, no external data file)
+COPY inline_model.py /tmp/
+RUN python /tmp/inline_model.py && rm /tmp/inline_model.py
+
+COPY server.py .
+
+EXPOSE 8080
+
+CMD ["python", "server.py"]

sibylline_scurl-0.2.3/cf/container/inline_model.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Download MiniLM model for prompt injection detection."""
+
+from huggingface_hub import hf_hub_download
+from pathlib import Path
+
+cache = '/root/.cache/scurl/models'
+
+# Download MiniLM-L6-v2 (86MB, single file - no external data)
+print('Downloading MiniLM-L6-v2 model...')
+model_path = hf_hub_download(
+    'Qdrant/all-MiniLM-L6-v2-onnx',
+    filename='model.onnx',
+    cache_dir=cache,
+)
+tokenizer_path = hf_hub_download(
+    'Qdrant/all-MiniLM-L6-v2-onnx',
+    filename='tokenizer.json',
+    cache_dir=cache,
+)
+
+print(f'Model downloaded to: {model_path}')
+print(f'Tokenizer downloaded to: {tokenizer_path}')
+print('Done')

sibylline_scurl-0.2.3/cf/container/server.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Simple HTTP server that runs scurl commands."""
+
+import json
+import subprocess
+import sys
+import tempfile
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+from scurl.sanitize import sanitize_text
+from scurl.middleware import ResponseContext
+
+TIMEOUT_SECONDS = 90
+MAX_CONVERT_SIZE = 256 * 1024  # 256 KB
+
+# Singleton defender — avoids re-loading the ONNX session on every request.
+# Safe because HTTPServer.serve_forever() is single-threaded.
+_defender = None
+
+
+def _get_defender(threshold: float = 0.3, action: str = "redact"):
+    """Return the singleton PromptInjectionDefender, creating it on first call."""
+    global _defender
+    if _defender is None:
+        from scurl.prompt_defender import PromptInjectionDefender
+        _defender = PromptInjectionDefender(
+            threshold=threshold,
+            action=action,
+        )
+    else:
+        if _defender.threshold != threshold:
+            _defender.threshold = threshold
+        if _defender.action != action:
+            _defender.action = action
+    return _defender
+
+
+def _injection_dict(analysis) -> dict:
+    """Convert an InjectionAnalysis into a JSON-safe dict."""
+    active_signals = [k for k, v in analysis.pattern_features.items() if v > 0]
+    return {
+        "score": round(float(analysis.score), 4),
+        "flagged": analysis.flagged,
+        "threshold": analysis.threshold,
+        "action_taken": analysis.action_taken,
+        "signals": active_signals,
+    }
+
+
+class ScurlHandler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers.get("Content-Length", 0))
+        body = self.rfile.read(content_length)
+
+        if self.path == "/convert":
+            self._handle_convert(body)
+        else:
+            self._handle_fetch(body)
+
+    def _handle_fetch(self, body: bytes):
+        try:
+            data = json.loads(body)
+            url = data.get("url")
+            render = data.get("render", True)
+            threshold = float(data.get("threshold", 0.3))
+            action = data.get("action", "redact")
+
+            if not url:
+                self._send_json(400, {"markdown": None, "error": "Missing 'url' field", "injection": None})
+                return
+
+            cmd = ["scurl", "--render", url] if render else ["scurl", url]
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=TIMEOUT_SECONDS,
+            )
+
+            if result.returncode == 0:
+                defender = _get_defender(threshold=threshold, action=action)
+                analysis = defender.analyze(result.stdout)
+                injection = _injection_dict(analysis)
+
+                ctx = ResponseContext(
+                    body=result.stdout.encode("utf-8"),
+                    headers={},
+                    status_code=200,
+                    content_type="text/plain",
+                    url=url,
+                )
+                if analysis.flagged and defender.should_process(ctx):
+                    processed = defender.process(ctx)
+                    output = processed.body.decode("utf-8", errors="replace")
+                else:
+                    output = result.stdout
+
+                self._send_json(200, {"markdown": output, "error": None, "injection": injection})
+            else:
+                error = result.stderr.strip() or f"scurl exited with code {result.returncode}"
+                self._send_json(500, {"markdown": None, "error": error, "injection": None})
+
+        except json.JSONDecodeError:
+            self._send_json(400, {"markdown": None, "error": "Invalid JSON", "injection": None})
+        except subprocess.TimeoutExpired:
+            self._send_json(504, {"markdown": None, "error": "Request timed out", "injection": None})
+        except Exception as e:
+            self._send_json(500, {"markdown": None, "error": str(e), "injection": None})
+
+    def _handle_convert(self, body: bytes):
+        try:
+            data = json.loads(body)
+            text = data.get("html") or data.get("text")
+            action = data.get("action", "redact")
+            threshold = float(data.get("threshold", 0.3))
+
+            if not text:
+                self._send_json(400, {"markdown": None, "error": "Missing 'html' or 'text' field", "injection": None})
+                return
+
+            text_bytes = len(text.encode("utf-8"))
+            if text_bytes > MAX_CONVERT_SIZE:
+                self._send_json(413, {
+                    "markdown": None,
+                    "error": f"Input too large ({text_bytes // 1024} KB). Maximum is 256 KB.",
+                    "injection": None,
+                })
+                return
+
+            # First sanitize (strip HTML, normalize whitespace)
+            cleaned = sanitize_text(text)
+
+            # Reuse singleton defender (avoids re-loading ONNX session)
+            defender = _get_defender(threshold=threshold, action=action)
+            analysis = defender.analyze(cleaned)
+            injection = _injection_dict(analysis)
+
+            ctx = ResponseContext(
+                body=cleaned.encode("utf-8"),
+                headers={},
+                status_code=200,
+                content_type="text/plain",
+                url="",
+            )
+
+            if analysis.flagged and defender.should_process(ctx):
+                result = defender.process(ctx)
+                output = result.body.decode("utf-8", errors="replace")
+            else:
+                output = cleaned
+
+            self._send_json(200, {"markdown": output, "error": None, "injection": injection})
+
+        except json.JSONDecodeError:
+            self._send_json(400, {"markdown": None, "error": "Invalid JSON", "injection": None})
+        except Exception as e:
+            self._send_json(500, {"markdown": None, "error": str(e), "injection": None})
+
+    def do_GET(self):
+        if self.path == "/health":
+            self._send_json(200, {"status": "ok"})
+        else:
+            self._send_json(404, {"error": "Not found"})
+
+    def _send_json(self, status: int, data: dict):
+        response = json.dumps(data)
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(response)))
+        self.end_headers()
+        self.wfile.write(response.encode())
+
+    def log_message(self, format, *args):
+        print(f"[scurl-server] {args[0]}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    # Eagerly load the defender + ONNX model so the cost is paid at boot,
+    # not on the first request.
+    print("[scurl-server] Pre-loading prompt injection model…", file=sys.stderr)
+    try:
+        defender = _get_defender()
+        defender._ensure_heavy_components()
+        defender._embedder.embed("warmup")
+        print("[scurl-server] Model ready", file=sys.stderr)
+    except Exception as e:
+        print(f"[scurl-server] Model pre-load failed (will retry on first request): {e}", file=sys.stderr)
+
+    server = HTTPServer(("0.0.0.0", 8080), ScurlHandler)
+    print("[scurl-server] Listening on port 8080", file=sys.stderr)
+    server.serve_forever()