sibylline-scurl 0.1.0__tar.gz

sibylline_scurl-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]" pytest-mock
+
+      - name: Run tests
+        run: pytest -v
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install ruff
+        run: pip install ruff
+
+      - name: Run ruff
+        run: ruff check src/ tests/

sibylline_scurl-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,71 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Install from wheel
+        run: pip install dist/*.whl
+
+      - name: Test CLI
+        run: |
+          scurl --help
+          scurl --list-middleware
+
+  publish-pypi:
+    runs-on: ubuntu-latest
+    needs: [build, test]
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

sibylline_scurl-0.1.0/.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db

sibylline_scurl-0.1.0/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.4
+Name: sibylline-scurl
+Version: 0.1.0
+Summary: A secure curl wrapper with middleware support and HTML-to-markdown extraction
+Author: Nathan
+License: MIT
+Keywords: curl,markdown,security,web-scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Security
+Requires-Python: >=3.10
+Requires-Dist: trafilatura>=1.6.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scurl
+
+[![PyPI version](https://badge.fury.io/py/scurl.svg)](https://badge.fury.io/py/scurl)
+[![CI](https://github.com/yourusername/scurl/actions/workflows/ci.yml/badge.svg)](https://github.com/yourusername/scurl/actions/workflows/ci.yml)
+
+A secure curl wrapper with middleware support and HTML-to-markdown extraction.
+
+## Installation
+
+```bash
+pip install scurl
+```
+
+Or with [pipx](https://pipx.pypa.io/) (recommended for CLI tools):
+
+```bash
+pipx install scurl
+```
+
+## Usage
+
+```bash
+# Fetch a URL and extract clean markdown from HTML
+scurl https://example.com
+
+# Raw output (disable response middleware)
+scurl --raw https://example.com
+
+# All curl flags work
+scurl -H "Accept: application/json" https://api.example.com/data
+```
+
+## Features
+
+- **SecretDefender**: Automatically detects and blocks requests containing exposed secrets/tokens
+- **TrafilaturaExtractor**: Extracts clean markdown from HTML responses
+- **Middleware System**: Composable request and response middleware
+
+## Flags
+
+| Flag | Description |
+|------|-------------|
+| `--raw` | Disable all response middleware |
+| `--disable <slug>` | Disable a middleware by slug (can be repeated) |
+| `--enable <slug>` | Override a middleware's block (can be repeated) |
+| `--list-middleware` | List available middleware and their slugs |
+
+## Middleware Slugs
+
+| Slug | Type | Description |
+|------|------|-------------|
+| `secret-defender` | Request | Detects and blocks requests containing secrets |
+| `trafilatura` | Response | Extracts clean markdown from HTML |
+
+## License
+
+MIT

sibylline_scurl-0.1.0/README.md

@@ -0,0 +1,57 @@
+# scurl
+
+[![PyPI version](https://badge.fury.io/py/scurl.svg)](https://badge.fury.io/py/scurl)
+[![CI](https://github.com/yourusername/scurl/actions/workflows/ci.yml/badge.svg)](https://github.com/yourusername/scurl/actions/workflows/ci.yml)
+
+A secure curl wrapper with middleware support and HTML-to-markdown extraction.
+
+## Installation
+
+```bash
+pip install scurl
+```
+
+Or with [pipx](https://pipx.pypa.io/) (recommended for CLI tools):
+
+```bash
+pipx install scurl
+```
+
+## Usage
+
+```bash
+# Fetch a URL and extract clean markdown from HTML
+scurl https://example.com
+
+# Raw output (disable response middleware)
+scurl --raw https://example.com
+
+# All curl flags work
+scurl -H "Accept: application/json" https://api.example.com/data
+```
+
+## Features
+
+- **SecretDefender**: Automatically detects and blocks requests containing exposed secrets/tokens
+- **TrafilaturaExtractor**: Extracts clean markdown from HTML responses
+- **Middleware System**: Composable request and response middleware
+
+## Flags
+
+| Flag | Description |
+|------|-------------|
+| `--raw` | Disable all response middleware |
+| `--disable <slug>` | Disable a middleware by slug (can be repeated) |
+| `--enable <slug>` | Override a middleware's block (can be repeated) |
+| `--list-middleware` | List available middleware and their slugs |
+
+## Middleware Slugs
+
+| Slug | Type | Description |
+|------|------|-------------|
+| `secret-defender` | Request | Detects and blocks requests containing secrets |
+| `trafilatura` | Response | Extracts clean markdown from HTML |
+
+## License
+
+MIT

sibylline_scurl-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "sibylline-scurl"
+version = "0.1.0"
+description = "A secure curl wrapper with middleware support and HTML-to-markdown extraction"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Nathan" }]
+keywords = ["curl", "web-scraping", "security", "markdown"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Security",
+]
+dependencies = [
+    "trafilatura>=1.6.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+scurl = "scurl.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scurl"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

sibylline_scurl-0.1.0/src/scurl/__init__.py

@@ -0,0 +1,3 @@
+"""scurl - A secure curl wrapper with middleware support."""
+
+__version__ = "0.1.0"

sibylline_scurl-0.1.0/src/scurl/cli.py

@@ -0,0 +1,194 @@
+"""CLI entry point for scurl."""
+
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+
+from .middleware import (
+    RequestMiddlewareChain,
+    ResponseMiddlewareChain,
+    RequestAction,
+)
+from .request_middleware import SecretDefender
+from .response_middleware import TrafilaturaExtractor
+from .curl import parse_curl_args, execute_curl, curl_result_to_response_context
+
+
+# Registry of available middleware with their slugs
+REQUEST_MIDDLEWARE = {
+    "secret-defender": ("SecretDefender", "Detects and blocks requests containing secrets", SecretDefender),
+}
+
+RESPONSE_MIDDLEWARE = {
+    "trafilatura": ("TrafilaturaExtractor", "Extracts clean markdown from HTML", TrafilaturaExtractor),
+}
+
+
+def print_middleware_list() -> None:
+    """Print available middleware."""
+    print("Request Middleware:")
+    for slug, (name, desc, _) in REQUEST_MIDDLEWARE.items():
+        print(f"  {slug:<20} {name} - {desc}")
+    print()
+    print("Response Middleware:")
+    for slug, (name, desc, _) in RESPONSE_MIDDLEWARE.items():
+        print(f"  {slug:<20} {name} - {desc}")
+
+
+@dataclass
+class ScurlFlags:
+    """Parsed scurl-specific flags."""
+    raw: bool = False
+    disable: set[str] = field(default_factory=set)
+    enable: set[str] = field(default_factory=set)
+    list_middleware: bool = False
+    help: bool = False
+
+
+def extract_scurl_flags(args: list[str]) -> tuple[ScurlFlags, list[str]]:
+    """Extract scurl-specific flags from args, return (flags, remaining_args)."""
+    flags = ScurlFlags()
+    remaining = []
+
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if arg == "--raw":
+            flags.raw = True
+            i += 1
+        elif arg == "--disable":
+            if i + 1 < len(args):
+                flags.disable.add(args[i + 1])
+                i += 2
+            else:
+                remaining.append(arg)
+                i += 1
+        elif arg == "--enable":
+            if i + 1 < len(args):
+                flags.enable.add(args[i + 1])
+                i += 2
+            else:
+                remaining.append(arg)
+                i += 1
+        elif arg == "--list-middleware":
+            flags.list_middleware = True
+            i += 1
+        elif arg in ("--help", "-h") and i == 0:
+            # Only treat as scurl help if it's the first arg
+            flags.help = True
+            i += 1
+        else:
+            remaining.append(arg)
+            i += 1
+
+    return flags, remaining
+
+
+def print_help() -> None:
+    """Print scurl help."""
+    print("scurl - A secure curl wrapper with middleware support")
+    print()
+    print("Usage: scurl [scurl-options] [curl-options] <url>")
+    print()
+    print("scurl-specific options:")
+    print("  --raw                  Disable all response middleware (raw curl output)")
+    print("  --disable <middleware>  Disable a middleware by slug (can be repeated)")
+    print("  --enable <middleware>  Override a middleware's block (can be repeated)")
+    print("  --list-middleware      List available middleware and their slugs")
+    print("  --help, -h             Show this help (use curl --help for curl options)")
+    print()
+    print("All other options are passed directly to curl.")
+    print()
+    print("Examples:")
+    print("  scurl https://example.com                    # Fetch and extract markdown")
+    print("  scurl --raw https://example.com              # Raw HTML output")
+    print("  scurl --disable trafilatura https://example.com # Disable markdown extraction")
+    print("  scurl --disable secret-defender https://...     # Disable secret scanning")
+    print("  scurl --enable secret-defender https://...   # Override a secret block")
+    print("  scurl -H 'Accept: application/json' https://api.example.com/data")
+
+
+def run(args: Optional[list[str]] = None) -> int:
+    """Run scurl with the given arguments. Returns exit code."""
+    if args is None:
+        args = sys.argv[1:]
+
+    # Extract scurl-specific flags
+    flags, curl_args = extract_scurl_flags(args)
+
+    if flags.help:
+        print_help()
+        return 0
+
+    if flags.list_middleware:
+        print_middleware_list()
+        return 0
+
+    if not curl_args:
+        print("scurl: no URL specified", file=sys.stderr)
+        print("Try 'scurl --help' for more information.", file=sys.stderr)
+        return 1
+
+    # Parse curl args to get request context
+    context = parse_curl_args(curl_args)
+
+    if not context.url:
+        print("scurl: no URL specified", file=sys.stderr)
+        return 1
+
+    # Build request middleware chain
+    request_chain = RequestMiddlewareChain()
+    secret_defender_enabled = "secret-defender" not in flags.disable
+    secret_defender_override = "secret-defender" in flags.enable
+
+    if secret_defender_enabled and not secret_defender_override:
+        request_chain.add(SecretDefender())
+
+    # Execute request middleware
+    result = request_chain.execute(context)
+    if result.action == RequestAction.BLOCK:
+        print(f"scurl: {result.reason}", file=sys.stderr)
+        return 1
+
+    # Use potentially modified context
+    if result.context:
+        context = result.context
+
+    # Execute curl
+    curl_result = execute_curl(context)
+
+    if curl_result.return_code != 0 and curl_result.return_code != -1:
+        # curl failed but not our timeout/not-found
+        if curl_result.stderr:
+            print(curl_result.stderr, file=sys.stderr)
+        return curl_result.return_code
+
+    if curl_result.return_code == -1:
+        print(f"scurl: {curl_result.stderr}", file=sys.stderr)
+        return 1
+
+    # Build response middleware chain
+    response_chain = ResponseMiddlewareChain()
+    if not flags.raw:
+        if "trafilatura" not in flags.disable:
+            response_chain.add(TrafilaturaExtractor())
+
+    # Execute response middleware
+    response_context = curl_result_to_response_context(curl_result)
+    response_result = response_chain.execute(response_context)
+
+    # Output result
+    sys.stdout.buffer.write(response_result.body)
+    if response_result.body and not response_result.body.endswith(b"\n"):
+        sys.stdout.buffer.write(b"\n")
+
+    return 0
+
+
+def main() -> None:
+    """Main entry point."""
+    sys.exit(run())
+
+
+if __name__ == "__main__":
+    main()