public-source-extractor 0.1.0a2__py3-none-any.whl

public_source_extractor/__init__.py

@@ -0,0 +1,3 @@
+"""Public Source Extractor package."""
+
+__version__ = "0.1.0a2"

public_source_extractor/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+raise SystemExit(main())
+

public_source_extractor/cli.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Sequence
+
+from . import __version__
+from .errors import ExtractorError, InputRejected
+from .output import build_success_envelope, render_json, render_markdown, write_new_file_atomic
+from .provider import FirecrawlKeylessProvider
+from .url_policy import validate_public_url
+
+
+class StableArgumentParser(argparse.ArgumentParser):
+    def error(self, _message: str) -> None:
+        raise InputRejected()
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = StableArgumentParser(
+        prog="public-source-extractor",
+        description="Convert one public URL into Markdown or a stable JSON envelope.",
+    )
+    parser.add_argument("url", help="Public HTTP or HTTPS URL.")
+    parser.add_argument("--mode", choices=("markdown", "json"), default="markdown")
+    parser.add_argument("--output", type=Path, help="Write to a new file instead of stdout.")
+    parser.add_argument("--timeout", type=int, default=60, metavar="SECONDS")
+    parser.add_argument("--provider", choices=("firecrawl-keyless",), default="firecrawl-keyless")
+    parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON output.")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser
+
+
+def run(argv: Sequence[str] | None = None) -> int:
+    try:
+        args = build_parser().parse_args(argv)
+        if not 1 <= args.timeout <= 120:
+            raise InputRejected()
+        validate_public_url(args.url)
+        provider = FirecrawlKeylessProvider()
+        result = provider.extract(args.url, args.mode, args.timeout)
+        envelope = build_success_envelope(args.url, args.mode, result)
+        rendered = (
+            render_markdown(envelope)
+            if args.mode == "markdown"
+            else render_json(envelope, args.pretty)
+        )
+        if args.output is None:
+            sys.stdout.write(rendered)
+        else:
+            write_new_file_atomic(args.output, rendered)
+        return 0
+    except ExtractorError as exc:
+        sys.stderr.write(json.dumps(exc.envelope(), ensure_ascii=False) + "\n")
+        return exc.exit_code
+
+
+def main() -> int:
+    return run()

public_source_extractor/errors.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+
+class ExtractorError(Exception):
+    code = "extractor_error"
+    exit_code = 1
+    retryable = False
+    public_message = "Extraction failed."
+
+    def __init__(self, message: str | None = None) -> None:
+        super().__init__(message or self.public_message)
+
+    def envelope(self) -> dict[str, object]:
+        return {
+            "schema_version": "0.1",
+            "ok": False,
+            "error": {
+                "code": self.code,
+                "message": self.public_message,
+                "retryable": self.retryable,
+            },
+        }
+
+
+class InputRejected(ExtractorError):
+    code = "input_rejected"
+    exit_code = 2
+    public_message = "The URL was rejected by the public-source safety policy."
+
+
+class ProviderFailure(ExtractorError):
+    code = "provider_failure"
+    exit_code = 3
+    retryable = True
+    public_message = "The extraction provider is unavailable or rejected the request."
+
+
+class ProviderRateLimited(ProviderFailure):
+    code = "provider_rate_limited"
+    public_message = "The experimental provider rate limit was exceeded."
+
+
+class ProviderTimeout(ProviderFailure):
+    code = "provider_timeout"
+    public_message = "The extraction provider timed out."
+
+
+class InvalidProviderResponse(ExtractorError):
+    code = "invalid_provider_response"
+    exit_code = 4
+    public_message = "The extraction provider returned an invalid or unsafe response."
+
+
+class OutputFailure(ExtractorError):
+    code = "output_failure"
+    exit_code = 5
+    public_message = "The output file could not be written safely."
+

public_source_extractor/output.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .errors import OutputFailure
+from .provider import FirecrawlKeylessProvider, ProviderResult
+
+
+def build_success_envelope(
+    requested_url: str,
+    mode: str,
+    result: ProviderResult,
+) -> dict[str, Any]:
+    warnings = [
+        {
+            "code": "experimental_provider",
+            "message": "Keyless availability and limits are not guaranteed.",
+        }
+    ]
+    if result.resolved_url is None:
+        warnings.append(
+            {
+                "code": "resolved_url_unavailable",
+                "message": "The provider did not return redirect metadata for post-checking.",
+            }
+        )
+    return {
+        "schema_version": "0.1",
+        "ok": True,
+        "source": {
+            "requested_url": requested_url,
+            "resolved_url": result.resolved_url,
+            "fetched_at": datetime.now(timezone.utc).isoformat(timespec="seconds").replace(
+                "+00:00", "Z"
+            ),
+        },
+        "mode": mode,
+        "content": result.content,
+        "metadata": {
+            "title": result.title,
+            "description": result.description,
+            "content_type": result.content_type,
+            "source_http_status": result.source_http_status,
+        },
+        "provider": {
+            "name": FirecrawlKeylessProvider.name,
+            "access": FirecrawlKeylessProvider.access,
+            "http_status": result.provider_http_status,
+            "credits_used": result.credits_used,
+            "elapsed_ms": result.elapsed_ms,
+        },
+        "warnings": warnings,
+    }
+
+
+def render_markdown(envelope: dict[str, Any]) -> str:
+    source = envelope["source"]
+    metadata = envelope["metadata"]
+    provider = envelope["provider"]
+    front_matter = {
+        "schema_version": envelope["schema_version"],
+        "source_url": source["requested_url"],
+        "resolved_url": source["resolved_url"],
+        "fetched_at": source["fetched_at"],
+        "title": metadata["title"],
+        "content_type": metadata["content_type"],
+        "source_http_status": metadata["source_http_status"],
+        "provider": provider["name"],
+        "provider_access": provider["access"],
+        "provider_credits_used": provider["credits_used"],
+        "provider_elapsed_ms": provider["elapsed_ms"],
+    }
+    lines = ["---"]
+    lines.extend(
+        f"{key}: {json.dumps(value, ensure_ascii=False)}" for key, value in front_matter.items()
+    )
+    lines.extend(["---", "", str(envelope["content"]), ""])
+    return "\n".join(lines)
+
+
+def render_json(envelope: dict[str, Any], pretty: bool) -> str:
+    indent = 2 if pretty else None
+    return json.dumps(envelope, ensure_ascii=False, indent=indent, separators=None) + "\n"
+
+
+def write_new_file_atomic(path: Path, content: str) -> None:
+    try:
+        parent = path.parent.resolve(strict=True)
+        _reject_symlink_components(path.parent)
+        if not parent.is_dir() or path.exists() or path.is_symlink():
+            raise OutputFailure()
+
+        fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=parent)
+        temporary = Path(temporary_name)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+                handle.write(content)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.link(temporary, path)
+        finally:
+            temporary.unlink(missing_ok=True)
+    except OutputFailure:
+        raise
+    except (OSError, RuntimeError) as exc:
+        raise OutputFailure() from exc
+
+
+def _reject_symlink_components(path: Path) -> None:
+    # Inspect the caller-selected path without resolving it first. macOS exposes
+    # /tmp and /var as stable platform aliases, so only those two are exempt.
+    platform_aliases = {Path("/tmp"), Path("/var")}
+    absolute = path.absolute()
+    current = Path(absolute.anchor)
+    for component in absolute.parts[1:]:
+        current /= component
+        if current.is_symlink() and current not in platform_aliases:
+            raise OutputFailure()

public_source_extractor/provider.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import json
+import socket
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from .errors import (
+    InputRejected,
+    InvalidProviderResponse,
+    ProviderFailure,
+    ProviderRateLimited,
+    ProviderTimeout,
+)
+from .url_policy import validate_provider_resolved_url
+
+
+FIRECRAWL_ENDPOINT = "https://api.firecrawl.dev/v2/scrape"
+MAX_RESPONSE_BYTES = 10 * 1024 * 1024
+
+STRUCTURED_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "title": {"type": "string"},
+        "summary": {"type": "string"},
+        "main_argument": {"type": "string"},
+        "key_points": {"type": "array", "items": {"type": "string"}},
+        "external_sources": {"type": "array", "items": {"type": "string"}},
+    },
+    "required": ["title", "summary", "main_argument"],
+}
+
+
+@dataclass(frozen=True)
+class ProviderResult:
+    content: str | dict[str, Any]
+    resolved_url: str | None
+    title: str | None
+    description: str | None
+    content_type: str | None
+    source_http_status: int | None
+    provider_http_status: int
+    credits_used: int | None
+    elapsed_ms: int
+
+
+class ExtractionProvider(Protocol):
+    def extract(self, url: str, mode: str, timeout_seconds: int) -> ProviderResult: ...
+
+
+class FirecrawlKeylessProvider:
+    name = "firecrawl-keyless"
+    access = "experimental"
+
+    def extract(self, url: str, mode: str, timeout_seconds: int) -> ProviderResult:
+        formats: list[Any]
+        if mode == "markdown":
+            formats = ["markdown"]
+        else:
+            formats = [{"type": "json", "schema": STRUCTURED_SCHEMA}]
+        payload = {
+            "url": url,
+            "formats": formats,
+            "onlyMainContent": True,
+            "removeBase64Images": True,
+            "timeout": timeout_seconds * 1000,
+        }
+        body = json.dumps(payload, ensure_ascii=True).encode("utf-8")
+        request = urllib.request.Request(
+            FIRECRAWL_ENDPOINT,
+            data=body,
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": "public-source-extractor/0.1.0-alpha.1",
+            },
+            method="POST",
+        )
+
+        started = time.monotonic()
+        try:
+            with urllib.request.urlopen(request, timeout=timeout_seconds + 5) as response:
+                status = response.status
+                raw = response.read(MAX_RESPONSE_BYTES + 1)
+        except urllib.error.HTTPError as exc:
+            if exc.code == 429:
+                raise ProviderRateLimited() from exc
+            raise ProviderFailure() from exc
+        except (urllib.error.URLError, TimeoutError, socket.timeout) as exc:
+            reason = getattr(exc, "reason", None)
+            if isinstance(exc, (TimeoutError, socket.timeout)) or isinstance(
+                reason, (TimeoutError, socket.timeout)
+            ):
+                raise ProviderTimeout() from exc
+            raise ProviderFailure() from exc
+
+        elapsed_ms = round((time.monotonic() - started) * 1000)
+        if len(raw) > MAX_RESPONSE_BYTES:
+            raise InvalidProviderResponse()
+        try:
+            document = json.loads(raw.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise InvalidProviderResponse() from exc
+        if status != 200 or not isinstance(document, dict) or document.get("success") is not True:
+            raise ProviderFailure()
+
+        data = document.get("data")
+        if not isinstance(data, dict):
+            raise InvalidProviderResponse()
+        metadata = data.get("metadata")
+        if metadata is None:
+            metadata = {}
+        if not isinstance(metadata, dict):
+            raise InvalidProviderResponse()
+
+        try:
+            resolved_url = validate_provider_resolved_url(
+                metadata.get("sourceURL") or metadata.get("url")
+            )
+        except InputRejected as exc:
+            raise InvalidProviderResponse() from exc
+        content = data.get("markdown") if mode == "markdown" else data.get("json")
+        if mode == "markdown" and not isinstance(content, str):
+            raise InvalidProviderResponse()
+        if mode == "json" and not isinstance(content, dict):
+            raise InvalidProviderResponse()
+
+        source_status = metadata.get("statusCode")
+        if not isinstance(source_status, int) or isinstance(source_status, bool):
+            source_status = None
+        credits_used = metadata.get("creditsUsed")
+        if not isinstance(credits_used, int) or isinstance(credits_used, bool):
+            credits_used = None
+
+        return ProviderResult(
+            content=content,
+            resolved_url=resolved_url,
+            title=_optional_string(metadata.get("title") or metadata.get("og:title")),
+            description=_optional_string(
+                metadata.get("description") or metadata.get("og:description")
+            ),
+            content_type=_optional_string(metadata.get("contentType")),
+            source_http_status=source_status,
+            provider_http_status=status,
+            credits_used=credits_used,
+            elapsed_ms=elapsed_ms,
+        )
+
+
+def _optional_string(value: object) -> str | None:
+    return value if isinstance(value, str) else None

public_source_extractor/schemas/output-v0.1.schema.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Ishikawa-Hidekazu/public-source-extractor/blob/main/schemas/output-v0.1.schema.json",
+  "title": "Public Source Extractor Output v0.1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "ok"],
+  "properties": {
+    "schema_version": {"const": "0.1"},
+    "ok": {"type": "boolean"},
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["requested_url", "resolved_url", "fetched_at"],
+      "properties": {
+        "requested_url": {"type": "string", "format": "uri"},
+        "resolved_url": {"type": ["string", "null"], "format": "uri"},
+        "fetched_at": {"type": "string", "format": "date-time"}
+      }
+    },
+    "mode": {"enum": ["markdown", "json"]},
+    "content": {"oneOf": [{"type": "string"}, {"type": "object"}]},
+    "metadata": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["title", "description", "content_type", "source_http_status"],
+      "properties": {
+        "title": {"type": ["string", "null"]},
+        "description": {"type": ["string", "null"]},
+        "content_type": {"type": ["string", "null"]},
+        "source_http_status": {"type": ["integer", "null"], "minimum": 100, "maximum": 599}
+      }
+    },
+    "provider": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "access", "http_status", "credits_used", "elapsed_ms"],
+      "properties": {
+        "name": {"type": "string", "const": "firecrawl-keyless", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"},
+        "access": {"const": "experimental"},
+        "http_status": {"type": "integer", "minimum": 100, "maximum": 599},
+        "credits_used": {"type": ["integer", "null"], "minimum": 0},
+        "elapsed_ms": {"type": "integer", "minimum": 0}
+      }
+    },
+    "warnings": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["code", "message"],
+        "properties": {
+          "code": {"type": "string", "pattern": "^[a-z0-9_]+$"},
+          "message": {"type": "string"}
+        }
+      }
+    },
+    "error": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "message", "retryable"],
+      "properties": {
+        "code": {"type": "string", "pattern": "^[a-z0-9_]+$"},
+        "message": {"type": "string"},
+        "retryable": {"type": "boolean"}
+      }
+    }
+  },
+  "allOf": [
+    {
+      "if": {"properties": {"ok": {"const": true}}, "required": ["ok"]},
+      "then": {
+        "required": ["source", "mode", "content", "metadata", "provider", "warnings"],
+        "not": {"required": ["error"]}
+      },
+      "else": {
+        "required": ["error"],
+        "not": {"anyOf": [{"required": ["content"]}, {"required": ["provider"]}]}
+      }
+    },
+    {
+      "if": {"properties": {"ok": {"const": true}, "mode": {"const": "markdown"}}, "required": ["ok", "mode"]},
+      "then": {"properties": {"content": {"type": "string"}}}
+    },
+    {
+      "if": {"properties": {"ok": {"const": true}, "mode": {"const": "json"}}, "required": ["ok", "mode"]},
+      "then": {"properties": {"content": {"type": "object"}}}
+    }
+  ]
+}

public_source_extractor/url_policy.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import ipaddress
+import re
+import urllib.parse
+
+from .errors import InputRejected
+
+
+BLOCKED_HOSTS = {
+    "accounts.google.com",
+    "calendar.google.com",
+    "docs.google.com",
+    "drive.google.com",
+    "mail.google.com",
+    "search.google.com",
+    "twitter.com",
+    "x.com",
+}
+
+LOCAL_SUFFIXES = (".local", ".localhost", ".internal", ".home", ".lan")
+
+BLOCKED_PATH_RE = re.compile(
+    r"(^|/)(wp-admin|wp-login\.php|admin|login|logout|oauth|callback)(/|$)",
+    re.IGNORECASE,
+)
+
+DENIED_QUERY_NAMES = {
+    "access_key",
+    "access_token",
+    "api_key",
+    "apikey",
+    "auth",
+    "authorization",
+    "code",
+    "cookie",
+    "credential",
+    "credentials",
+    "key",
+    "passwd",
+    "password",
+    "refresh_token",
+    "secret",
+    "session",
+    "sessionid",
+    "token",
+}
+
+DENIED_QUERY_SEGMENTS = {
+    "auth",
+    "authorization",
+    "cookie",
+    "credential",
+    "credentials",
+    "key",
+    "passwd",
+    "password",
+    "secret",
+    "session",
+    "signature",
+    "token",
+}
+
+
+def _decode_repeated(value: str, rounds: int = 3) -> str:
+    decoded = value
+    for _ in range(rounds):
+        next_value = urllib.parse.unquote(decoded)
+        if next_value == decoded:
+            break
+        decoded = next_value
+    return decoded
+
+
+def _query_name_is_sensitive(name: str) -> bool:
+    decoded = _decode_repeated(name).lower()
+    canonical = re.sub(r"[^a-z0-9]+", "_", decoded).strip("_")
+    if canonical in DENIED_QUERY_NAMES:
+        return True
+    segments = {segment for segment in canonical.split("_") if segment}
+    return bool(segments & DENIED_QUERY_SEGMENTS)
+
+
+def _validate_host(host: str) -> None:
+    if not host or any(ord(char) > 127 for char in host):
+        raise InputRejected()
+    if "%" in host or "\\" in host:
+        raise InputRejected()
+
+    normalized = host.lower().rstrip(".")
+    if normalized in {"localhost", "local"} or normalized.endswith(LOCAL_SUFFIXES):
+        raise InputRejected()
+    if normalized in BLOCKED_HOSTS or any(
+        normalized.endswith(f".{blocked}") for blocked in BLOCKED_HOSTS
+    ):
+        raise InputRejected()
+
+    try:
+        ip = ipaddress.ip_address(normalized)
+    except ValueError:
+        ip = None
+
+    if ip is not None:
+        if not ip.is_global:
+            raise InputRejected()
+        return
+
+    # Reject legacy integer, octal, and hex IPv4 spellings instead of relying on resolver behavior.
+    if re.fullmatch(r"[0-9.]+", normalized) or normalized.startswith(("0x", "0X")):
+        raise InputRejected()
+    if not re.fullmatch(r"[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?", normalized):
+        raise InputRejected()
+    if ".." in normalized or "." not in normalized:
+        raise InputRejected()
+
+
+def validate_public_url(raw_url: str) -> urllib.parse.ParseResult:
+    if not raw_url or any(char in raw_url for char in ("\r", "\n", "\x00")):
+        raise InputRejected()
+
+    parsed = urllib.parse.urlparse(raw_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc or not parsed.hostname:
+        raise InputRejected()
+    if parsed.username or parsed.password or parsed.fragment:
+        raise InputRejected()
+
+    try:
+        port = parsed.port
+    except ValueError as exc:
+        raise InputRejected() from exc
+    expected_port = 443 if parsed.scheme == "https" else 80
+    if port is not None and port != expected_port:
+        raise InputRejected()
+
+    raw_host = parsed.netloc.rsplit("@", 1)[-1]
+    if raw_host.startswith("["):
+        raw_host = raw_host[1 : raw_host.find("]")]
+    else:
+        raw_host = raw_host.rsplit(":", 1)[0] if ":" in raw_host else raw_host
+    if "%" in raw_host or any(ord(char) > 127 for char in raw_host):
+        raise InputRejected()
+    _validate_host(parsed.hostname)
+
+    decoded_path = _decode_repeated(parsed.path or "/").replace("\\", "/")
+    if BLOCKED_PATH_RE.search(decoded_path):
+        raise InputRejected()
+
+    try:
+        query_pairs = urllib.parse.parse_qsl(
+            parsed.query,
+            keep_blank_values=True,
+            strict_parsing=False,
+            max_num_fields=100,
+        )
+    except ValueError as exc:
+        raise InputRejected() from exc
+    for name, _value in query_pairs:
+        if _query_name_is_sensitive(name):
+            raise InputRejected()
+
+    return parsed
+
+
+def validate_provider_resolved_url(raw_url: object) -> str | None:
+    if raw_url is None:
+        return None
+    if not isinstance(raw_url, str):
+        raise InputRejected()
+    validate_public_url(raw_url)
+    return raw_url

public_source_extractor-0.1.0a2.dist-info/METADATA

@@ -0,0 +1,230 @@
+Metadata-Version: 2.4
+Name: public-source-extractor
+Version: 0.1.0a2
+Summary: Validate a public URL and convert it into reviewable AI-ready Markdown or JSON.
+Author: Ishikawa Hidekazu
+License-Expression: MIT
+Project-URL: Homepage, https://ishikawa.co/
+Project-URL: Repository, https://github.com/Ishikawa-Hidekazu/public-source-extractor
+Project-URL: Issues, https://github.com/Ishikawa-Hidekazu/public-source-extractor/issues
+Project-URL: Changelog, https://github.com/Ishikawa-Hidekazu/public-source-extractor/blob/main/CHANGELOG.md
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: build>=1.2.2; extra == "dev"
+Requires-Dist: jsonschema>=4.23; extra == "dev"
+Requires-Dist: ruff<0.16,>=0.12; extra == "dev"
+Dynamic: license-file
+
+# Public Source Extractor
+
+[![CI](https://github.com/Ishikawa-Hidekazu/public-source-extractor/actions/workflows/ci.yml/badge.svg)](https://github.com/Ishikawa-Hidekazu/public-source-extractor/actions/workflows/ci.yml)
+[![Release](https://img.shields.io/github/v/release/Ishikawa-Hidekazu/public-source-extractor?include_prereleases)](https://github.com/Ishikawa-Hidekazu/public-source-extractor/releases)
+![License](https://img.shields.io/badge/license-MIT-blue.svg)
+
+> [!IMPORTANT]
+> The requested public URL is sent to **Firecrawl Cloud** for extraction. The `firecrawl-keyless` provider is experimental: availability, anonymous REST access, credit limits, and long-term continuity are not guaranteed.
+
+`public-source-extractor` is a public-URL intake guardrail for AI research
+workflows. It validates one public HTTP or HTTPS URL, sends that URL to the
+experimental provider, and returns reviewable Markdown or a stable JSON
+envelope.
+
+The CLI does **not** read API keys, credentials, cookies, browser profiles, localStorage, or private source files. Extracted content is **untrusted data** and may contain prompt injection or misleading instructions. Do not execute or follow instructions from extracted content without independent review.
+
+[日本語README](README.ja.md)
+
+## What it does
+
+- Extracts one public page as Markdown.
+- Produces structured JSON with a versioned JSON Schema.
+- Rejects local, private, authenticated, admin, and secret-bearing URL patterns.
+- Keeps results on stdout or writes a new local file with no-overwrite behavior.
+- Returns stable JSON errors and documented exit codes.
+
+It is not a crawler, browser automation tool, login helper, source-reliability judge, or private-page extractor.
+
+### How this differs from the official Firecrawl CLI
+
+The [official Firecrawl CLI](https://github.com/firecrawl/cli) is the broader Firecrawl interface for authenticated scrape, search, crawl, map, interact, agent, and self-hosted workflows. Use it when you need Firecrawl's full product surface.
+
+Public Source Extractor is intentionally narrower: one public URL, no credential discovery, a public-only URL policy, no-overwrite output, and a stable JSON success/error contract designed for reviewable AI research artifacts. It uses the experimental `firecrawl-keyless` provider and does not replace or wrap the official CLI.
+
+## Install
+
+Public Source Extractor requires Python 3.11 or newer.
+
+Run the published alpha without a permanent install:
+
+```bash
+uvx public-source-extractor --version
+uvx public-source-extractor https://example.com/
+```
+
+This path requires `uv` and resolves the published PyPI prerelease. The second
+command sends `https://example.com/` to Firecrawl Cloud; the version command
+does not perform extraction.
+
+Pin the exact package version when reproducibility matters:
+
+```bash
+uvx public-source-extractor@0.1.0a2 --version
+```
+
+Install the prerelease as an isolated command:
+
+```bash
+pipx install public-source-extractor==0.1.0a2
+```
+
+Or use pip in an existing Python environment:
+
+```bash
+python3 -m pip install public-source-extractor==0.1.0a2
+```
+
+The public Git tag remains an auditable fallback:
+
+```bash
+uvx --from 'git+https://github.com/Ishikawa-Hidekazu/public-source-extractor.git@v0.1.0-alpha.2' public-source-extractor --version
+```
+
+PyPI publication uses GitHub Actions Trusted Publishing with short-lived OIDC
+credentials. No long-lived PyPI API token is stored in this repository.
+
+## Quick start
+
+Markdown to stdout:
+
+```bash
+public-source-extractor https://example.com/
+```
+
+Markdown front matter includes `provider_credits_used` and
+`provider_elapsed_ms`. The credits field is `null` when the experimental
+provider does not report it; elapsed time is measured by the CLI in
+milliseconds. These are metadata-only values. Raw provider responses and
+request identifiers are not exposed.
+
+JSON to stdout:
+
+```bash
+public-source-extractor https://example.com/ --mode json --pretty
+```
+
+Write a new file:
+
+```bash
+public-source-extractor https://example.com/ --output report.md
+```
+
+The output path must have an existing non-symlink parent and must not already exist.
+
+<picture>
+  <source media="(max-width: 600px)" srcset="assets/source/terminal-example-mobile.svg">
+  <img src="assets/source/terminal-example.svg" alt="Fixture-only Public Source Extractor terminal example showing one public example.com URL converted to Markdown, followed by the stable provider_rate_limited JSON error contract.">
+</picture>
+
+[View the public-safe examples](examples/) ·
+[View the reproducible visual sources](assets/source/README.md)
+
+## Real-world use
+
+[This Japanese implementation log](https://taupe.site/entry/public-source-extractor-ai-research-cli/) shows the CLI used to turn selected public sources into reviewable Markdown and JSON artifacts. It also documents the Firecrawl Cloud boundary, untrusted extracted content, and observed provider limits.
+
+## CLI contract
+
+```text
+public-source-extractor <url> [--mode markdown|json]
+                               [--output <new-path>]
+                               [--timeout 1..120]
+                               [--provider firecrawl-keyless]
+                               [--pretty]
+```
+
+| Exit | Meaning |
+|---:|---|
+| `0` | Success |
+| `2` | Usage error or rejected URL |
+| `3` | Provider, network, rate-limit, or timeout failure |
+| `4` | Invalid, incomplete, or unsafe provider response |
+| `5` | Output path or write failure |
+
+On failure, stdout is empty and stderr contains exactly one JSON error object. Provider response bodies, stack traces, request IDs, and local paths are not exposed.
+
+### Recovering from `provider_rate_limited`
+
+The experimental provider can return HTTP 429 during short bursts or across an anonymous usage window. This is a provider availability condition, not by itself a failure of the local URL policy or parser.
+
+When stderr reports `provider_rate_limited` with `retryable: true`:
+
+```json
+{"schema_version":"0.1","ok":false,"error":{"code":"provider_rate_limited","message":"The experimental provider rate limit was exceeded.","retryable":true}}
+```
+
+The process exits with code `3` and stdout remains empty.
+
+1. Stop the current burst instead of retrying repeatedly.
+2. Wait and retry later. This CLI does not promise an exact delay when no retry timing is safely exposed.
+3. Reduce request volume and process only selected public sources.
+4. Use the original public page directly when extraction is not required.
+
+The CLI does not automatically retry, switch providers, discover credentials, or expose raw provider bodies. See [Issue #9](https://github.com/Ishikawa-Hidekazu/public-source-extractor/issues/9) for the observed condition and documentation scope.
+
+## Safety boundary
+
+Before sending a URL, the CLI rejects:
+
+- schemes other than HTTP or HTTPS;
+- URL user information and fragments;
+- localhost, local suffixes, and non-global literal IP addresses;
+- ambiguous integer, octal, and hexadecimal IPv4 forms;
+- Unicode or percent-encoded hostnames and IPv6 zone identifiers;
+- non-default ports;
+- login, admin, OAuth, and callback paths, including encoded forms;
+- query parameter names that indicate tokens, secrets, passwords, credentials, signatures, sessions, cookies, authorization, or keys.
+
+After extraction, provider redirect metadata is checked with the same public URL policy. If redirect metadata is unsafe, the result is discarded. If the provider does not supply redirect metadata, the output contains a warning.
+
+These checks validate hostname syntax, literal IP policy, and provider-returned metadata. They cannot fully guarantee DNS rebinding behavior or the provider's actual fetch destination. Never put credentials, tokens, signed URL parameters, or other private values in a URL, even when its hostname looks public.
+
+Firecrawl Cloud receives the requested URL and processes the page. Review [Firecrawl's terms](https://www.firecrawl.dev/terms-of-service), the target site's terms, robots policy, copyright, and privacy requirements before use.
+
+## JSON Schema and examples
+
+- [Output Schema v0.1](schemas/output-v0.1.schema.json)
+- [Public-safe Markdown example](examples/example-report.md)
+- [Public-safe JSON example](examples/example-report.json)
+
+Structured JSON extraction is inferred output, not source truth. Validate important claims against the original page.
+
+## Development
+
+```bash
+python3 -m venv .venv
+.venv/bin/python -m pip install -e '.[dev]'
+PYTHONPATH=src .venv/bin/python -m unittest discover -s tests -v
+.venv/bin/ruff check src tests
+.venv/bin/python -m build
+```
+
+The explicit `PYTHONPATH=src` also works before an editable install. Network smoke tests remain separate from the offline suite.
+
+See [CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and [SUPPORT.md](SUPPORT.md).
+
+## Status
+
+Alpha package. Package version `0.1.0a2` maps to tag `v0.1.0-alpha.2`.
+The package is distributed through PyPI and the matching public Git tag.
+`firecrawl-keyless` is an experimental third-party provider, and no
+compatibility or service-availability guarantee is made.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

public_source_extractor-0.1.0a2.dist-info/RECORD

@@ -0,0 +1,15 @@
+public_source_extractor/__init__.py,sha256=DXZn-eFYumzPbGxLy9c1IKRjV74ha9D9R-hiGbsY6lo,64
+public_source_extractor/__main__.py,sha256=4BTAfHHF2u3j_oUg_Wluqu0HyxnXfAHBQUcnPQ-e1xs,50
+public_source_extractor/cli.py,sha256=83rLUPa5UwP4WmO-lBeaPPNxGriYAOyJZQLsI6dP88E,2265
+public_source_extractor/errors.py,sha256=CM1hD09Z6LpuW-9tAltt_LOUN3y-ulteNorzNhcZTHc,1594
+public_source_extractor/output.py,sha256=WfouveBMseAzvUtZRgYA9HFhgjQDGqNQXbaNZ6jSidI,4201
+public_source_extractor/provider.py,sha256=vkxAQnJF8qYVP7ry8D--hBtQqwS6rVl8Bu-yPjKfwlg,5348
+public_source_extractor/url_policy.py,sha256=ypmyDR_BmOC1iF1CppaoFgH1lGmmY9DzwGoxKOoV1nU,4574
+public_source_extractor/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+public_source_extractor/schemas/output-v0.1.schema.json,sha256=g_jI026fIlkEv9xbwlOtL05-9rCTz1rAeu1GmIYRJ7I,3288
+public_source_extractor-0.1.0a2.dist-info/licenses/LICENSE,sha256=GIuyYJ3MmxBeSj1qQof3BPeEmm_vrdY74hNw621LX_g,1075
+public_source_extractor-0.1.0a2.dist-info/METADATA,sha256=GY8AJd-oVWA2mdI1cq1RhcyI3lKuG8O0_arhOBG6PTQ,10174
+public_source_extractor-0.1.0a2.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+public_source_extractor-0.1.0a2.dist-info/entry_points.txt,sha256=gFp0UdMNcmUWP3Kn4oAshglkyKskYJbXec6rWxp8RMc,77
+public_source_extractor-0.1.0a2.dist-info/top_level.txt,sha256=xeq1Zecr5hCHGHjWY3sGm4m07RYbwtNvHuQstpaR6TM,24
+public_source_extractor-0.1.0a2.dist-info/RECORD,,

public_source_extractor-0.1.0a2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

public_source_extractor-0.1.0a2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+public-source-extractor = public_source_extractor.cli:main

public_source_extractor-0.1.0a2.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Ishikawa Hidekazu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

public_source_extractor-0.1.0a2.dist-info/top_level.txt

@@ -0,0 +1 @@
+public_source_extractor