PyPI - render-url - Versions diffs - 1.0.0__py3-none-any.whl - Mend

render-url 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (8) hide show

parse_html.py +432 -0
render_and_parse.py +156 -0
render_url-1.0.0.dist-info/METADATA +189 -0
render_url-1.0.0.dist-info/RECORD +8 -0
render_url-1.0.0.dist-info/WHEEL +5 -0
render_url-1.0.0.dist-info/entry_points.txt +4 -0
render_url-1.0.0.dist-info/top_level.txt +3 -0
render_url.py +322 -0

parse_html.py ADDED Viewed

@@ -0,0 +1,432 @@
+"""
+parse_html.py — deterministic HTML -> structured JSON extractor (Step 2).
+Pipeline: Step 1 JSON (rendered_page_*.json) -> "html" field -> BeautifulSoup4
+          (html.parser backend) -> deterministic structural extraction rules
+          -> single JSON object on stdout.
+No LLM, AI model, agent reasoning, or semantic HTML interpretation is used
+anywhere in this script. No network access, no Playwright. This module is
+fully independent of render_url.py (never imports it).
+Extracted fields (see parse_html()):
+  title    <title> text, whitespace-normalized. null if absent/empty.
+  headings every h1-h6 in document order: {"level": int, "text": str}.
+  links    every <a href=...> in document order: {"text": str, "href": str}.
+           Anchors without an href attribute are excluded. Not deduped.
+  images   every <img src=...> in document order: {"src": str, "alt": str}.
+           alt defaults to "" if the attribute is absent. Images without a
+           src attribute are excluded.
+  meta     dict of {name_or_property: content} from <meta name=...> and
+           <meta property=...> tags, collected in document order with the
+           later tag winning on duplicate keys.
+  text     visible text of <body> (or whole document if no body), with
+           <script>/<style> contents excluded, whitespace-normalized.
+`selector` (optional CSS selector, config key) scopes headings/links/images/
+text extraction to the first element matched by BeautifulSoup's .select();
+title/meta remain document-level. No match -> those fields come back empty
+and `selector_matched` is false (not an error).
+`strip_whitespace` (default true) controls whitespace normalization
+(collapse runs of whitespace to a single space, strip ends) for text/
+headings/links text. When false, raw tag text is used as-is.
+"""
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from bs4 import BeautifulSoup
+DEFAULT_SELECTOR = None
+DEFAULT_STRIP_WHITESPACE = True
+DEFAULT_OUTPUT_PREFIX = "parsed_page"
+DEFAULT_OUTPUT_DIR = "."
+DEFAULT_CONFIG_PATH = "config.json"
+DEFAULT_VERBOSE = False
+DEFAULT_LOG_JSON = False
+DEFAULTS = {
+    "selector": DEFAULT_SELECTOR,
+    "strip_whitespace": DEFAULT_STRIP_WHITESPACE,
+    "output_prefix": DEFAULT_OUTPUT_PREFIX,
+    "output_dir": DEFAULT_OUTPUT_DIR,
+    "verbose": DEFAULT_VERBOSE,
+    "log_json": DEFAULT_LOG_JSON,
+}
+def _log(verbose, message):
+    if verbose:
+        print(f"[parse-html] {message}", file=sys.stderr)
+_WHITESPACE_RE = re.compile(r"\s+")
+def load_config_file(path):
+    """Read the shared config file and return only its "parser" sub-object.
+    render_url.py owns the rest of the file (flat renderer keys); this
+    reads the same file but scopes itself to the "parser" key so both
+    tools can coexist in one config.json with no changes needed to
+    render_url.py's own key handling beyond tolerating "parser" as known-but-
+    ignored.
+    """
+    p = Path(path)
+    if not p.exists():
+        return {}
+    try:
+        with p.open(encoding="utf-8") as f:
+            data = json.load(f)
+    except (OSError, json.JSONDecodeError) as e:
+        raise SystemExit(f"error: failed to read config file {path}: {e}")
+    if not isinstance(data, dict):
+        raise SystemExit(f"error: config file {path} must contain a JSON object.")
+    parser_section = data.get("parser", {})
+    if not isinstance(parser_section, dict):
+        raise SystemExit(f"error: \"parser\" section in {path} must be a JSON object.")
+    unknown = set(parser_section) - set(DEFAULTS)
+    if unknown:
+        raise SystemExit(
+            f"error: unknown parser config key(s) in {path}: {', '.join(sorted(unknown))}"
+        )
+    return parser_section
+def resolve_settings(args):
+    """Merge defaults <- config file's "parser" section <- CLI args (CLI wins)."""
+    settings = dict(DEFAULTS)
+    settings.update(load_config_file(args.config))
+    for key in DEFAULTS:
+        cli_value = getattr(args, key, None)
+        if cli_value is not None:
+            settings[key] = cli_value
+    return settings
+def next_output_path(prefix, directory="."):
+    n = 1
+    while True:
+        candidate = Path(directory) / f"{prefix}_{n}.json"
+        if not candidate.exists():
+            return candidate
+        n += 1
+def build_result(ok, source_url=None, title=None, data=None, selector_matched=None, error=None):
+    return {
+        "ok": ok,
+        "source_url": source_url,
+        "title": title,
+        "data": data,
+        "selector_matched": selector_matched,
+        "error": error,
+    }
+def error_result(source_url, error_type, message):
+    return build_result(ok=False, source_url=source_url, error={"type": error_type, "message": message})
+def _norm(text, strip_whitespace):
+    if strip_whitespace:
+        return _WHITESPACE_RE.sub(" ", text).strip()
+    return text
+def _extract_title(soup, strip_whitespace):
+    tag = soup.find("title")
+    if tag is None:
+        return None
+    text = _norm(tag.get_text(), strip_whitespace)
+    return text if text else None
+def _extract_headings(scope, strip_whitespace):
+    headings = []
+    for tag in scope.find_all(re.compile(r"^h[1-6]$")):
+        level = int(tag.name[1])
+        headings.append({"level": level, "text": _norm(tag.get_text(), strip_whitespace)})
+    return headings
+def _extract_links(scope, strip_whitespace):
+    links = []
+    for tag in scope.find_all("a"):
+        if not tag.has_attr("href"):
+            continue
+        text = _norm(tag.get_text(), strip_whitespace) if tag.get_text() else ""
+        links.append({"text": text, "href": tag.get("href")})
+    return links
+def _extract_images(scope):
+    images = []
+    for tag in scope.find_all("img"):
+        if not tag.has_attr("src"):
+            continue
+        images.append({"src": tag.get("src"), "alt": tag.get("alt", "")})
+    return images
+def _extract_meta(soup):
+    meta = {}
+    for tag in soup.find_all("meta"):
+        key = tag.get("property") if tag.has_attr("property") else tag.get("name")
+        if not key or not tag.has_attr("content"):
+            continue
+        meta[key] = tag.get("content")
+    return meta
+def _extract_text(scope, strip_whitespace):
+    for tag in scope.find_all(["script", "style"]):
+        tag.decompose()
+    text = scope.get_text(separator=" ")
+    return _norm(text, strip_whitespace)
+def parse_html(html, config=None):
+    """Parse raw HTML into the deterministic structured-extraction schema.
+    Returns the same dict shape build_result() produces. No network access,
+    no Playwright, no LLM/AI reasoning; pure structural DOM traversal.
+    """
+    config = config or {}
+    source_url = config.get("source_url")
+    strip_whitespace = config.get("strip_whitespace", DEFAULT_STRIP_WHITESPACE)
+    selector = config.get("selector", DEFAULT_SELECTOR)
+    verbose = config.get("verbose", DEFAULT_VERBOSE)
+    if html is None or not isinstance(html, str):
+        _log(verbose, "invalid input: html is missing or not a string")
+        return error_result(source_url, "invalid_input", "html must be a non-empty string.")
+    try:
+        if not isinstance(strip_whitespace, bool):
+            raise ValueError("strip_whitespace must be a boolean.")
+        if selector is not None and not isinstance(selector, str):
+            raise ValueError("selector must be a string or null.")
+    except ValueError as e:
+        _log(verbose, f"invalid config: {e}")
+        return error_result(source_url, "invalid_config", str(e))
+    _log(verbose, f"parsing {len(html)} chars of HTML with BeautifulSoup4 (html.parser)")
+    try:
+        soup = BeautifulSoup(html, "html.parser")
+    except Exception as e:
+        _log(verbose, f"parse error: {e}")
+        return error_result(source_url, "parse_error", str(e))
+    try:
+        _log(verbose, "extracting title and meta tags")
+        title = _extract_title(soup, strip_whitespace)
+        meta = _extract_meta(soup)
+        selector_matched = None
+        scope = soup.body if soup.body is not None else soup
+        if selector:
+            _log(verbose, f"applying selector: {selector!r}")
+            try:
+                matches = soup.select(selector)
+            except Exception as e:
+                _log(verbose, f"invalid selector: {e}")
+                return error_result(source_url, "invalid_config", f"invalid selector: {e}")
+            if matches:
+                scope = matches[0]
+                selector_matched = True
+                _log(verbose, "selector matched an element; scoping extraction to it")
+            else:
+                selector_matched = False
+                _log(verbose, "selector matched nothing; returning empty scoped fields")
+                return build_result(
+                    ok=True,
+                    source_url=source_url,
+                    title=title,
+                    data={"headings": [], "links": [], "images": [], "meta": meta, "text": ""},
+                    selector_matched=False,
+                    error=None,
+                )
+        _log(verbose, "extracting headings, links, images, and text")
+        data = {
+            "headings": _extract_headings(scope, strip_whitespace),
+            "links": _extract_links(scope, strip_whitespace),
+            "images": _extract_images(scope),
+            "meta": meta,
+            "text": _extract_text(scope, strip_whitespace),
+        }
+        _log(
+            verbose,
+            f"done: {len(data['headings'])} headings, {len(data['links'])} links, "
+            f"{len(data['images'])} images extracted",
+        )
+        return build_result(
+            ok=True,
+            source_url=source_url,
+            title=title,
+            data=data,
+            selector_matched=selector_matched,
+            error=None,
+        )
+    except Exception as e:
+        _log(verbose, f"unexpected error: {e}")
+        return error_result(source_url, "unknown_error", str(e))
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Parse a Step 1 rendered-page JSON file's HTML into deterministic structured JSON."
+    )
+    parser.add_argument(
+        "input",
+        nargs="?",
+        default=None,
+        help="Path to a Step 1 JSON output file (or use --input). Use \"-\" to read from stdin.",
+    )
+    parser.add_argument(
+        "--input",
+        dest="input_opt",
+        default=None,
+        help="Path to a Step 1 JSON output file (alternative to the positional argument).",
+    )
+    parser.add_argument(
+        "--config",
+        default=DEFAULT_CONFIG_PATH,
+        help=f"Path to a JSON config file (default: {DEFAULT_CONFIG_PATH}). Reads its \"parser\" section.",
+    )
+    parser.add_argument(
+        "--selector",
+        dest="selector",
+        default=None,
+        help="CSS selector to scope headings/links/images/text extraction to.",
+    )
+    parser.add_argument(
+        "--no-strip-whitespace",
+        dest="strip_whitespace",
+        action="store_const",
+        const=False,
+        default=None,
+        help="Preserve original whitespace in extracted text instead of collapsing it.",
+    )
+    parser.add_argument(
+        "--output-prefix",
+        dest="output_prefix",
+        default=None,
+        help=f"Prefix for the auto-incremented output file (default: {DEFAULT_OUTPUT_PREFIX}).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        dest="output_dir",
+        default=None,
+        help=f"Directory to write the output file into (default: {DEFAULT_OUTPUT_DIR}).",
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        dest="verbose",
+        action="store_const",
+        const=True,
+        default=None,
+        help="Log parsing progress (input loading, extraction steps, field counts) to stderr.",
+    )
+    parser.add_argument(
+        "--log-json",
+        dest="log_json",
+        action="store_const",
+        const=True,
+        default=None,
+        help="Additionally log the final result JSON, pretty-printed, to stderr.",
+    )
+    return parser.parse_args(argv)
+def _print_result(result, log_json):
+    print(json.dumps(result))
+    if log_json:
+        print(json.dumps(result, indent=2), file=sys.stderr)
+def main():
+    args = parse_args()
+    settings = resolve_settings(args)
+    verbose = settings["verbose"]
+    log_json = settings["log_json"]
+    input_path = args.input_opt or args.input
+    if not input_path:
+        _log(verbose, "no input file provided")
+        result = error_result(None, "invalid_input", "No input file provided (positional argument, --input, or \"-\" for stdin).")
+        _print_result(result, log_json)
+        sys.exit(1)
+    _log(verbose, f"reading input from {'stdin' if input_path == '-' else input_path}")
+    try:
+        if input_path == "-":
+            raw_text = sys.stdin.read()
+        else:
+            raw_text = Path(input_path).read_text(encoding="utf-8")
+    except OSError as e:
+        _log(verbose, f"failed to read input: {e}")
+        result = error_result(None, "invalid_input", f"failed to read input file {input_path}: {e}")
+        _print_result(result, log_json)
+        sys.exit(1)
+    try:
+        step1 = json.loads(raw_text)
+    except json.JSONDecodeError as e:
+        _log(verbose, f"input is not valid JSON: {e}")
+        result = error_result(None, "invalid_input", f"input file is not valid JSON: {e}")
+        _print_result(result, log_json)
+        output_path = next_output_path(settings["output_prefix"], settings["output_dir"])
+        _write_output(output_path, result, verbose)
+        sys.exit(1)
+    if not isinstance(step1, dict):
+        _log(verbose, "input JSON is not an object")
+        result = error_result(None, "invalid_input", "Step 1 JSON must be an object.")
+        _print_result(result, log_json)
+        output_path = next_output_path(settings["output_prefix"], settings["output_dir"])
+        _write_output(output_path, result, verbose)
+        sys.exit(1)
+    source_url = step1.get("url")
+    html = step1.get("html")
+    if not step1.get("ok", False) or html is None:
+        _log(verbose, "Step 1 result had ok=false or no html field; skipping parse")
+        result = error_result(source_url, "missing_html", "Step 1 result had ok=false or a null html field.")
+    else:
+        config = {
+            "source_url": source_url,
+            "selector": settings["selector"],
+            "strip_whitespace": settings["strip_whitespace"],
+            "verbose": verbose,
+        }
+        try:
+            result = parse_html(html, config=config)
+        except Exception as e:
+            _log(verbose, f"unexpected error: {e}")
+            result = error_result(source_url, "unknown_error", str(e))
+    _print_result(result, log_json)
+    output_path = next_output_path(settings["output_prefix"], settings["output_dir"])
+    _write_output(output_path, result, verbose)
+def _write_output(path, result, verbose=False):
+    _log(verbose, f"writing output to {path}")
+    try:
+        path.write_text(json.dumps(result), encoding="utf-8")
+    except OSError as e:
+        print(f"warning: failed to write output file {path}: {e}", file=sys.stderr)
+if __name__ == "__main__":
+    main()

render_and_parse.py ADDED Viewed

@@ -0,0 +1,156 @@
+"""
+render_and_parse.py — convenience wrapper chaining Step 1 and Step 2.
+Pipeline: URL -> render_url.render() -> Step 1 JSON -> parse_html.parse_html()
+          -> Step 2 JSON on stdout.
+This is a thin orchestration layer only. It imports both render_url and
+parse_html and calls their public functions directly (no subprocesses); it
+adds no rendering or parsing logic of its own. render_url.py and
+parse_html.py remain independently usable and parse_html.py still does not
+import render_url.
+"""
+import argparse
+import json
+import sys
+from types import SimpleNamespace
+import render_url
+import parse_html
+def _log(verbose, message):
+    if verbose:
+        print(f"[render-and-parse] {message}", file=sys.stderr)
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Render a URL with Chromium and parse the result into structured JSON, in one step."
+    )
+    parser.add_argument("url", nargs="?", default=None, help="URL to render and parse.")
+    parser.add_argument("--config", default=render_url.DEFAULT_CONFIG_PATH,
+                         help=f"Path to a JSON config file (default: {render_url.DEFAULT_CONFIG_PATH}).")
+    parser.add_argument("--timeout", dest="timeout_ms", type=int, default=None,
+                         help="Overall navigation timeout in milliseconds (render stage).")
+    parser.add_argument("--stabilization", dest="stabilization_ms", type=int, default=None,
+                         help="Settle time in milliseconds after load (render stage).")
+    parser.add_argument("--wait-until", dest="wait_until", choices=render_url.VALID_WAIT_UNTIL, default=None,
+                         help="Load state to wait for after navigation (render stage).")
+    parser.add_argument("--headless", dest="headless", action="store_const", const=True, default=None,
+                         help="Run Chromium headless (render stage).")
+    parser.add_argument("--no-headless", dest="headless", action="store_const", const=False,
+                         help="Run Chromium with a visible window (render stage).")
+    parser.add_argument("--selector", dest="selector", default=None,
+                         help="CSS selector to scope extraction to (parse stage).")
+    parser.add_argument("--no-strip-whitespace", dest="strip_whitespace", action="store_const",
+                         const=False, default=None, help="Preserve raw whitespace (parse stage).")
+    parser.add_argument("--output-dir", dest="output_dir", default=None,
+                         help="Directory for both the rendered_page_*.json and parsed_page_*.json files.")
+    parser.add_argument("--verbose", "-v", dest="verbose", action="store_const", const=True, default=None,
+                         help="Log progress from both the render and parse stages to stderr.")
+    parser.add_argument("--log-json", dest="log_json", action="store_const", const=True, default=None,
+                         help="Additionally log the render stage and final parse stage JSON, pretty-printed, to stderr.")
+    return parser.parse_args(argv)
+def _log_json(log_json, label, result):
+    if log_json:
+        print(f"[render-and-parse] {label}:", file=sys.stderr)
+        print(json.dumps(result, indent=2), file=sys.stderr)
+def main():
+    args = parse_args()
+    render_args = SimpleNamespace(
+        config=args.config,
+        url=args.url,
+        timeout_ms=args.timeout_ms,
+        stabilization_ms=args.stabilization_ms,
+        wait_until=args.wait_until,
+        headless=args.headless,
+        output_prefix=None,
+        output_dir=args.output_dir,
+        verbose=args.verbose,
+        log_json=args.log_json,
+    )
+    render_settings = render_url.resolve_settings(render_args)
+    verbose = render_settings["verbose"]
+    log_json = render_settings["log_json"]
+    url = render_settings["url"]
+    if not url:
+        result = parse_html.error_result(None, "invalid_input", "No URL provided via argument or config file.")
+        print(json.dumps(result))
+        _log_json(log_json, "final result", result)
+        sys.exit(1)
+    _log(verbose, f"stage 1/2: rendering {url}")
+    if not render_url.is_valid_url(url):
+        render_result = render_url.error_result(url, "invalid_url", "URL must be an absolute http(s) URL.")
+    else:
+        try:
+            render_result = render_url.render(
+                url,
+                timeout_ms=render_settings["timeout_ms"],
+                stabilization_ms=render_settings["stabilization_ms"],
+                wait_until=render_settings["wait_until"],
+                headless=render_settings["headless"],
+                verbose=verbose,
+            )
+        except Exception as e:
+            render_result = render_url.error_result(url, "unknown_error", str(e))
+    render_output_path = render_url.next_output_path(render_settings["output_prefix"], render_settings["output_dir"])
+    _log(verbose, f"writing render stage output to {render_output_path}")
+    _log_json(log_json, "render stage result", render_result)
+    try:
+        render_output_path.write_text(json.dumps(render_result), encoding="utf-8")
+    except OSError as e:
+        print(f"warning: failed to write output file {render_output_path}: {e}", file=sys.stderr)
+    parse_args_ns = SimpleNamespace(
+        config=args.config,
+        input=None,
+        input_opt=None,
+        selector=args.selector,
+        strip_whitespace=args.strip_whitespace,
+        output_prefix=None,
+        output_dir=args.output_dir,
+        verbose=args.verbose,
+        log_json=args.log_json,
+    )
+    parse_settings = parse_html.resolve_settings(parse_args_ns)
+    _log(verbose, "stage 2/2: parsing rendered HTML")
+    html = render_result.get("html")
+    if not render_result.get("ok", False) or html is None:
+        _log(verbose, "render stage failed or produced no html; skipping parse")
+        parse_result = parse_html.error_result(url, "missing_html", "Step 1 result had ok=false or a null html field.")
+    else:
+        parse_config = {
+            "source_url": render_result.get("url"),
+            "selector": parse_settings["selector"],
+            "strip_whitespace": parse_settings["strip_whitespace"],
+            "verbose": verbose,
+        }
+        try:
+            parse_result = parse_html.parse_html(html, config=parse_config)
+        except Exception as e:
+            parse_result = parse_html.error_result(url, "unknown_error", str(e))
+    parse_output_path = parse_html.next_output_path(parse_settings["output_prefix"], parse_settings["output_dir"])
+    _log(verbose, f"writing parse stage output to {parse_output_path}")
+    try:
+        parse_output_path.write_text(json.dumps(parse_result), encoding="utf-8")
+    except OSError as e:
+        print(f"warning: failed to write output file {parse_output_path}: {e}", file=sys.stderr)
+    print(json.dumps(parse_result))
+    _log_json(log_json, "final result", parse_result)
+if __name__ == "__main__":
+    main()

render_url-1.0.0.dist-info/METADATA ADDED Viewed

@@ -0,0 +1,189 @@
+Metadata-Version: 2.4
+Name: render-url
+Version: 1.0.0
+Summary: Render a single URL with Chromium (via Playwright) and emit the post-JS DOM as JSON.
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: beautifulsoup4>=4.12.0
+# PlaywrightURLJsonExtractor
+Two independent, deterministic CLI tools — no LLM/AI, no crawling:
+- **`render-url`** — renders one URL in headless Chromium (via Playwright) and outputs the post-JS DOM as JSON.
+- **`parse-html`** — parses the `html` field from `render-url`'s output into structured JSON (via BeautifulSoup4).
+```
+URL -> render-url -> rendered JSON (html field) -> parse-html -> structured JSON
+```
+## Install
+```bash
+pip install -e .
+playwright install chromium
+```
+This puts `render-url` and `parse-html` on your `PATH`.
+## Commands
+### `render-url` — render a URL to JSON
+```bash
+render-url "https://example.com"
+```
+Prints one JSON object to stdout and writes it to an auto-incremented file (`rendered_page_1.json`, `rendered_page_2.json`, ...). Never overwrites existing files.
+| Parameter | Flag | Default | Description |
+|---|---|---|---|
+| URL | positional, or `url` in config | *(none)* | The single URL to render. Required (from CLI or config). |
+| Config file | `--config <path>` | `config.json` | Path to the JSON config file. |
+| Timeout | `--timeout <ms>` | `30000` | Overall navigation timeout in milliseconds. |
+| Stabilization | `--stabilization <ms>` | `1000` | Fixed settle time (ms) after load, before capturing the DOM. |
+| Wait condition | `--wait-until <state>` | `load` | `load`, `domcontentloaded`, or `networkidle`. |
+| Headless | `--headless` / `--no-headless` | `--headless` | Run Chromium headless or with a visible window. |
+| Output prefix | `--output-prefix <name>` | `rendered_page` | Base name for the output JSON file. |
+| Output dir | `--output-dir <dir>` | `.` | Directory the output file is written into. |
+| Verbose | `--verbose` / `-v` | off | Log progress (navigation, waits, extraction, browser lifecycle) to stderr. Stdout still carries only the final JSON. |
+| Log JSON | `--log-json` | off | Additionally pretty-print the final result JSON to stderr. |
+### `parse-html` — extract structured data from rendered HTML
+```bash
+parse-html rendered_page_1.json
+```
+Reads a `render-url` JSON file (or stdin via `-`), extracts fields from its `html`, prints one JSON object to stdout, and writes it to an auto-incremented file (`parsed_page_1.json`, `parsed_page_2.json`, ...).
+| Parameter | Flag | Default | Description |
+|---|---|---|---|
+| Input file | positional, or `--input <path>` | *(none)* | `render-url` JSON file to read. Use `-` for stdin. |
+| Config file | `--config <path>` | `config.json` | Path to the JSON config file (reads its `"parser"` section). |
+| CSS selector | `--selector <css>` | `null` | Scopes `headings`/`links`/`images`/`text` to the first matching element. No match -> those fields come back empty, not an error. |
+| Whitespace | `--no-strip-whitespace` | strip on | Disable whitespace collapsing in extracted text. |
+| Output prefix | `--output-prefix <name>` | `parsed_page` | Base name for the output JSON file. |
+| Output dir | `--output-dir <dir>` | `.` | Directory the output file is written into. |
+| Verbose | `--verbose` / `-v` | off | Log progress (input loading, extraction steps, field counts) to stderr. Stdout still carries only the final JSON. |
+| Log JSON | `--log-json` | off | Additionally pretty-print the final result JSON to stderr. |
+### Extracted fields
+| Field | What it is | Fallback |
+|---|---|---|
+| `title` | `<title>` text | `null` if absent/empty |
+| `headings` | `<h1>`–`<h6>`, in order | `[]` if none |
+| `links` | `<a href>` text + href, in order (duplicates kept) | anchors without `href` excluded |
+| `images` | `<img src>` + `alt`, in order | `alt` defaults to `""`; no `src` excluded |
+| `meta` | `<meta name/property>` -> `content` | later tag wins on duplicate keys |
+| `text` | visible body text (scripts/styles excluded) | `""` if none |
+### `render-and-parse` — do both in one command
+```bash
+render-and-parse "https://example.com"
+```
+Runs `render-url` then `parse-html` in a single process and prints the final structured JSON. Writes both `rendered_page_N.json` and `parsed_page_N.json`.
+| Parameter | Flag | Default | Description |
+|---|---|---|---|
+| URL | positional | *(none)* | The URL to render and parse. |
+| Config file | `--config <path>` | `config.json` | Shared config file for both stages. |
+| Timeout | `--timeout <ms>` | `30000` | Render-stage navigation timeout. |
+| Stabilization | `--stabilization <ms>` | `1000` | Render-stage settle time. |
+| Wait condition | `--wait-until <state>` | `load` | Render-stage load state. |
+| Headless | `--headless` / `--no-headless` | `--headless` | Render-stage Chromium visibility. |
+| CSS selector | `--selector <css>` | `null` | Parse-stage extraction scope. |
+| Whitespace | `--no-strip-whitespace` | strip on | Parse-stage whitespace handling. |
+| Output dir | `--output-dir <dir>` | `.` | Directory for both output files. |
+| Verbose | `--verbose` / `-v` | off | Log progress from both stages to stderr. |
+| Log JSON | `--log-json` | off | Additionally pretty-print the render stage and final result JSON to stderr. |
+### Full pipeline manually (two commands)
+```bash
+render-url "https://example.com"
+parse-html rendered_page_1.json
+# or piped:
+render-url "https://example.com" | parse-html -
+```
+### Tests
+```bash
+python -m pytest tests/ -v
+```
+### Publishing to PyPI
+```bash
+pip install build twine
+# bump "version" in pyproject.toml first, then:
+rm -rf dist build render_url.egg-info    # PowerShell: Remove-Item -Recurse -Force dist, build, render_url.egg-info -ErrorAction SilentlyContinue
+python -m build                # builds dist/*.whl and dist/*.tar.gz
+python -m twine check dist/*   # validates metadata before upload
+python -m twine upload dist/*  # uploads to PyPI (prompts for credentials/token)
+```
+Use `python -m twine upload --repository testpypi dist/*` to publish to [TestPyPI](https://test.pypi.org/) first if you want to verify the package before a real release.
+## Configuration
+Precedence: **built-in defaults < `config.json` < CLI flags**. Both tools share one `config.json` (`parse-html` only reads its `"parser"` key). Copy [config.example.json](config.example.json) to get started — `config.json` is gitignored (local/per-environment). Unknown keys are rejected as typos.
+```json
+{
+  "url": "https://example.com",
+  "timeout_ms": 30000,
+  "stabilization_ms": 1000,
+  "wait_until": "load",
+  "headless": true,
+  "output_prefix": "rendered_page",
+  "output_dir": ".",
+  "verbose": false,
+  "log_json": false,
+  "parser": {
+    "selector": null,
+    "strip_whitespace": true,
+    "output_prefix": "parsed_page",
+    "output_dir": ".",
+    "verbose": false,
+    "log_json": false
+  }
+}
+```
+## Output schemas
+**`render-url` success:**
+```json
+{"ok": true, "url": "...", "final_url": "...", "status_code": 200, "title": "...", "html": "...", "error": null}
+```
+Errors: `invalid_url`, `navigation_error`, `navigation_timeout`, `browser_error`, `html_extraction_error`, `unknown_error`.
+**`parse-html` success:**
+```json
+{
+  "ok": true,
+  "source_url": "...",
+  "title": "...",
+  "data": {"headings": [...], "links": [...], "images": [...], "meta": {...}, "text": "..."},
+  "selector_matched": null,
+  "error": null
+}
+```
+Errors: `invalid_input`, `missing_html` (Step 1 failed or had no `html`), `invalid_config`, `parse_error`, `unknown_error`.
+Both tools always print exactly one JSON object to stdout (success or failure) — diagnostics go to stderr only.
+## What this project intentionally does NOT do
+- No crawling, link-following, or multi-URL batching.
+- No DOM mutation, clicking, form submission, or authentication.
+- No LLM, AI, embeddings, or semantic interpretation — structural extraction only.
+- `parse-html` makes no network calls and never launches a browser.

render_url-1.0.0.dist-info/RECORD ADDED Viewed

@@ -0,0 +1,8 @@
+parse_html.py,sha256=zvLD6Nrph0cutkuLSET4-NCPkev7-0x9uX-XiL6GOjY,15189
+render_and_parse.py,sha256=khjFlkSlnmaexchr5Tah0bi8KYPg3umsGKs8Tp5ObYs,6950
+render_url.py,sha256=raeb1uQwV4O6JX1rlPQqLvKVU3FyaFMRizWvyY_hvqA,10908
+render_url-1.0.0.dist-info/METADATA,sha256=7vzl0iEjrzNnhkBpqiyluW8uzXb5gtlUOTpElzag5fw,8212
+render_url-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+render_url-1.0.0.dist-info/entry_points.txt,sha256=1CIhkowLh4Uxh4k7oDCqfc97gRKiItv7QyjPStv9U58,117
+render_url-1.0.0.dist-info/top_level.txt,sha256=TNsFRwMotgU-rCoAsa1ZJNwOc23VnFeX1vSXHdhiwaQ,39
+render_url-1.0.0.dist-info/RECORD,,

render_url-1.0.0.dist-info/WHEEL ADDED Viewed

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any

render_url-1.0.0.dist-info/entry_points.txt ADDED Viewed

@@ -0,0 +1,4 @@
+[console_scripts]
+parse-html = parse_html:main
+render-and-parse = render_and_parse:main
+render-url = render_url:main

render_url-1.0.0.dist-info/top_level.txt ADDED Viewed

@@ -0,0 +1,3 @@
+parse_html
+render_and_parse
+render_url

render_url.py ADDED Viewed

@@ -0,0 +1,322 @@
+"""
+render_url.py — deterministic URL -> rendered HTML JSON utility.
+Pipeline: URL -> Playwright (Chromium) -> JS/WASM execution -> rendered DOM
+          -> HTML -> single JSON object on stdout.
+No LLM, AI model, agent reasoning, or semantic HTML interpretation is used
+anywhere in this script. It is a renderer, not a scraper.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+from playwright.sync_api import sync_playwright, Error as PlaywrightError, TimeoutError as PlaywrightTimeoutError
+DEFAULT_TIMEOUT_MS = 30000
+DEFAULT_STABILIZATION_MS = 1000
+DEFAULT_OUTPUT_PREFIX = "rendered_page"
+DEFAULT_OUTPUT_DIR = "."
+DEFAULT_WAIT_UNTIL = "load"
+DEFAULT_HEADLESS = True
+DEFAULT_CONFIG_PATH = "config.json"
+# wait_until values Playwright's page.wait_for_load_state() accepts.
+VALID_WAIT_UNTIL = ("load", "domcontentloaded", "networkidle")
+DEFAULT_VERBOSE = False
+DEFAULT_LOG_JSON = False
+DEFAULTS = {
+    "url": None,
+    "timeout_ms": DEFAULT_TIMEOUT_MS,
+    "stabilization_ms": DEFAULT_STABILIZATION_MS,
+    "wait_until": DEFAULT_WAIT_UNTIL,
+    "headless": DEFAULT_HEADLESS,
+    "output_prefix": DEFAULT_OUTPUT_PREFIX,
+    "output_dir": DEFAULT_OUTPUT_DIR,
+    "verbose": DEFAULT_VERBOSE,
+    "log_json": DEFAULT_LOG_JSON,
+}
+def load_config_file(path):
+    """Read a JSON config file. Returns {} if the file doesn't exist."""
+    p = Path(path)
+    if not p.exists():
+        return {}
+    try:
+        with p.open(encoding="utf-8") as f:
+            data = json.load(f)
+    except (OSError, json.JSONDecodeError) as e:
+        raise SystemExit(f"error: failed to read config file {path}: {e}")
+    if not isinstance(data, dict):
+        raise SystemExit(f"error: config file {path} must contain a JSON object.")
+    # "parser" is reserved for Step 2 (parse_html.py), which reads the same
+    # config file but only looks at this sub-key. Ignored here so both tools
+    # can share one config.json without render_url.py rejecting it as a typo.
+    unknown = set(data) - set(DEFAULTS) - {"parser"}
+    if unknown:
+        raise SystemExit(f"error: unknown config key(s) in {path}: {', '.join(sorted(unknown))}")
+    return {k: v for k, v in data.items() if k != "parser"}
+def resolve_settings(args):
+    """Merge defaults <- config file <- CLI args (CLI wins)."""
+    settings = dict(DEFAULTS)
+    settings.update(load_config_file(args.config))
+    for key in DEFAULTS:
+        cli_value = getattr(args, key, None)
+        if cli_value is not None:
+            settings[key] = cli_value
+    return settings
+def next_output_path(prefix, directory="."):
+    n = 1
+    while True:
+        candidate = Path(directory) / f"{prefix}_{n}.json"
+        if not candidate.exists():
+            return candidate
+        n += 1
+def build_result(ok, url, final_url=None, status_code=None, title=None, html=None, error=None):
+    return {
+        "ok": ok,
+        "url": url,
+        "final_url": final_url,
+        "status_code": status_code,
+        "title": title,
+        "html": html,
+        "error": error,
+    }
+def error_result(url, error_type, message):
+    return build_result(
+        ok=False,
+        url=url,
+        error={"type": error_type, "message": message},
+    )
+def is_valid_url(url):
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+def _log(verbose, message):
+    if verbose:
+        print(f"[render-url] {message}", file=sys.stderr)
+def render(url, timeout_ms, stabilization_ms=DEFAULT_STABILIZATION_MS,
+           wait_until=DEFAULT_WAIT_UNTIL, headless=DEFAULT_HEADLESS, verbose=False):
+    with sync_playwright() as p:
+        browser = None
+        try:
+            _log(verbose, f"launching Chromium (headless={headless})")
+            try:
+                browser = p.chromium.launch(headless=headless)
+            except PlaywrightError as e:
+                _log(verbose, f"browser launch failed: {e}")
+                return error_result(url, "browser_error", str(e))
+            page = browser.new_page()
+            page.set_default_timeout(timeout_ms)
+            status_code = None
+            _log(verbose, f"navigating to {url} (timeout={timeout_ms}ms)")
+            try:
+                response = page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
+                if response is not None:
+                    status_code = response.status
+                _log(verbose, f"domcontentloaded (status={status_code})")
+            except PlaywrightTimeoutError as e:
+                _log(verbose, f"navigation timed out: {e}")
+                return error_result(url, "navigation_timeout", str(e))
+            except PlaywrightError as e:
+                _log(verbose, f"navigation error: {e}")
+                return error_result(url, "navigation_error", str(e))
+            # Best-effort extra settle time beyond domcontentloaded, without
+            # relying on networkidle by default (SPAs may keep long-lived
+            # connections open and never reach it) — configurable via
+            # wait_until for callers who do want it.
+            _log(verbose, f"waiting for load state \"{wait_until}\"")
+            try:
+                page.wait_for_load_state(wait_until, timeout=timeout_ms)
+            except PlaywrightTimeoutError:
+                _log(verbose, f"load state \"{wait_until}\" timed out (continuing)")
+            except PlaywrightError:
+                pass
+            _log(verbose, f"stabilizing for {stabilization_ms}ms")
+            page.wait_for_timeout(stabilization_ms)
+            _log(verbose, "extracting outerHTML, title, and final URL")
+            try:
+                html = page.evaluate("document.documentElement.outerHTML")
+                title = page.title()
+                final_url = page.url
+            except PlaywrightError as e:
+                _log(verbose, f"HTML extraction failed: {e}")
+                return error_result(url, "html_extraction_error", str(e))
+            _log(verbose, f"done: {len(html)} chars of HTML captured")
+            return build_result(
+                ok=True,
+                url=url,
+                final_url=final_url,
+                status_code=status_code,
+                title=title,
+                html=html,
+                error=None,
+            )
+        finally:
+            if browser is not None:
+                _log(verbose, "closing Chromium")
+                try:
+                    browser.close()
+                except PlaywrightError:
+                    pass
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Render a single URL with Chromium (via Playwright) and emit the post-JS DOM as JSON."
+    )
+    parser.add_argument(
+        "url",
+        nargs="?",
+        default=None,
+        help="Exactly one URL to render. Falls back to \"url\" in the config file if omitted.",
+    )
+    parser.add_argument(
+        "--config",
+        default=DEFAULT_CONFIG_PATH,
+        help=f"Path to a JSON config file (default: {DEFAULT_CONFIG_PATH}; ignored if it doesn't exist). "
+             "CLI flags override values from this file.",
+    )
+    parser.add_argument(
+        "--timeout",
+        dest="timeout_ms",
+        type=int,
+        default=None,
+        help=f"Overall navigation timeout in milliseconds (default: {DEFAULT_TIMEOUT_MS}).",
+    )
+    parser.add_argument(
+        "--stabilization",
+        dest="stabilization_ms",
+        type=int,
+        default=None,
+        help=f"Fixed settle time after load, in milliseconds, before capturing the DOM "
+             f"(default: {DEFAULT_STABILIZATION_MS}).",
+    )
+    parser.add_argument(
+        "--wait-until",
+        dest="wait_until",
+        choices=VALID_WAIT_UNTIL,
+        default=None,
+        help=f"Load state to wait for after navigation (default: {DEFAULT_WAIT_UNTIL}). "
+             "\"networkidle\" is not recommended for SPAs that keep long-lived connections open.",
+    )
+    parser.add_argument(
+        "--headless",
+        dest="headless",
+        action="store_const",
+        const=True,
+        default=None,
+        help=f"Run Chromium headless (default: {DEFAULT_HEADLESS}).",
+    )
+    parser.add_argument(
+        "--no-headless",
+        dest="headless",
+        action="store_const",
+        const=False,
+        help="Run Chromium with a visible browser window.",
+    )
+    parser.add_argument(
+        "--output-prefix",
+        dest="output_prefix",
+        default=None,
+        help=f"Prefix for the auto-incremented output file, e.g. <prefix>_1.json, <prefix>_2.json, ... "
+             f"(default: {DEFAULT_OUTPUT_PREFIX}).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        dest="output_dir",
+        default=None,
+        help=f"Directory to write the output file into (default: {DEFAULT_OUTPUT_DIR}).",
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        dest="verbose",
+        action="store_const",
+        const=True,
+        default=None,
+        help="Log rendering progress (navigation, waits, extraction, browser lifecycle) to stderr.",
+    )
+    parser.add_argument(
+        "--log-json",
+        dest="log_json",
+        action="store_const",
+        const=True,
+        default=None,
+        help="Additionally log the final result JSON, pretty-printed, to stderr.",
+    )
+    return parser.parse_args(argv)
+def main():
+    args = parse_args()
+    settings = resolve_settings(args)
+    verbose = settings["verbose"]
+    log_json = settings["log_json"]
+    url = settings["url"]
+    if not url:
+        print(json.dumps(error_result(None, "invalid_url", "No URL provided via argument or config file.")))
+        sys.exit(1)
+    if not is_valid_url(url):
+        _log(verbose, f"invalid URL: {url!r}")
+        result = error_result(url, "invalid_url", "URL must be an absolute http(s) URL.")
+    else:
+        try:
+            result = render(
+                url,
+                timeout_ms=settings["timeout_ms"],
+                stabilization_ms=settings["stabilization_ms"],
+                wait_until=settings["wait_until"],
+                headless=settings["headless"],
+                verbose=verbose,
+            )
+        except Exception as e:
+            _log(verbose, f"unexpected error: {e}")
+            result = error_result(url, "unknown_error", str(e))
+    output_text = json.dumps(result)
+    print(output_text)
+    if log_json:
+        print(json.dumps(result, indent=2), file=sys.stderr)
+    output_path = next_output_path(settings["output_prefix"], settings["output_dir"])
+    _log(verbose, f"writing output to {output_path}")
+    try:
+        output_path.write_text(output_text, encoding="utf-8")
+    except OSError as e:
+        print(f"warning: failed to write output file {output_path}: {e}", file=sys.stderr)
+if __name__ == "__main__":
+    main()