scrape-cli 1.2.4__tar.gz → 1.3.1__tar.gz

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.2.4
+Version: 1.3.1
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -11,6 +11,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
+Requires-Dist: xmltodict
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)
@@ -71,6 +72,7 @@ pip install -e .
 - requests
 - lxml
 - cssselect
+- xmltodict
 
 ## How does it work?
 
@@ -230,13 +232,12 @@ scrape -te 'h1, h2, h3' resources/test.html
 
 The `-t` option automatically excludes text from `<script>` and `<style>` tags and cleans up whitespace for better readability.
 
-### JSON Output Integration
+### JSON Output
 
-You can integrate scrape-cli with [xq](https://github.com/kislyuk/yq) (part of yq) to convert HTML output to structured JSON:
+Use the `-j`/`--json` flag to get structured JSON natively, with no external tools:
 
 ```bash
-# Extract and convert to JSON (requires -b for complete HTML)
-scrape -be "a.external-link" resources/test.html | xq .
+scrape -je "a.external-link" resources/test.html
 ```
 
 Output:
@@ -258,7 +259,7 @@ Output:
 Table extraction example:
 
 ```bash
-scrape -be "table.data-table td" resources/test.html | xq .
+scrape -je "table.data-table td" resources/test.html
 ```
 
 Output:
@@ -268,19 +269,23 @@ Output:
   "html": {
     "body": {
       "td": [
-        "1",
-        "John Doe",
-        "john@example.com",
-        "2",
-        "Jane Smith",
-        "jane@example.com"
+        "Italy",
+        "Rome",
+        "59",
+        "France",
+        "Paris",
+        "68"
       ]
     }
   }
 }
 ```
 
-**Note**: The `-b` flag is mandatory to produce valid HTML with `<html>`, `<head>` and `<body>` tags.
+`-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
+
+`-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
+
+`-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.
 
 Useful for JSON-based pipelines, APIs, databases, and processing with jq/DuckDB.
 
@@ -288,6 +293,7 @@ Some notes on the commands:
 
 - `-e` to set the query
 - `-b` to add `<html>`, `<head>` and `<body>` tags to the HTML output
+- `-j` to output structured JSON (built-in)
 - `-t` to extract only text content (useful for LLMs and text processing)
 
 

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/README.md

@@ -57,6 +57,7 @@ pip install -e .
 - requests
 - lxml
 - cssselect
+- xmltodict
 
 ## How does it work?
 
@@ -216,13 +217,12 @@ scrape -te 'h1, h2, h3' resources/test.html
 
 The `-t` option automatically excludes text from `<script>` and `<style>` tags and cleans up whitespace for better readability.
 
-### JSON Output Integration
+### JSON Output
 
-You can integrate scrape-cli with [xq](https://github.com/kislyuk/yq) (part of yq) to convert HTML output to structured JSON:
+Use the `-j`/`--json` flag to get structured JSON natively, with no external tools:
 
 ```bash
-# Extract and convert to JSON (requires -b for complete HTML)
-scrape -be "a.external-link" resources/test.html | xq .
+scrape -je "a.external-link" resources/test.html
 ```
 
 Output:
@@ -244,7 +244,7 @@ Output:
 Table extraction example:
 
 ```bash
-scrape -be "table.data-table td" resources/test.html | xq .
+scrape -je "table.data-table td" resources/test.html
 ```
 
 Output:
@@ -254,19 +254,23 @@ Output:
   "html": {
     "body": {
       "td": [
-        "1",
-        "John Doe",
-        "john@example.com",
-        "2",
-        "Jane Smith",
-        "jane@example.com"
+        "Italy",
+        "Rome",
+        "59",
+        "France",
+        "Paris",
+        "68"
       ]
     }
   }
 }
 ```
 
-**Note**: The `-b` flag is mandatory to produce valid HTML with `<html>`, `<head>` and `<body>` tags.
+`-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
+
+`-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
+
+`-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.
 
 Useful for JSON-based pipelines, APIs, databases, and processing with jq/DuckDB.
 
@@ -274,6 +278,7 @@ Some notes on the commands:
 
 - `-e` to set the query
 - `-b` to add `<html>`, `<head>` and `<body>` tags to the HTML output
+- `-j` to output structured JSON (built-in)
 - `-t` to extract only text content (useful for LLMs and text processing)
 
 

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrape_cli"
-version = "1.2.4"
+version = "1.3.1"
 description = "It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector."
 readme = "README.md"
 authors = [
@@ -18,7 +18,8 @@ requires-python = ">=3.8"
 dependencies = [
     "cssselect",
     "lxml",
-    "requests"
+    "requests",
+    "xmltodict"
 ]
 
 [project.urls]

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/scrape_cli/__init__.py

@@ -4,7 +4,7 @@ scrape-cli - A command-line tool to extract HTML elements using XPath or CSS3 se
 
 from scrape_cli.scrape import main
 
-__version__ = "1.2.4"
+__version__ = "1.3.1"
 __author__ = "Andrea Borruso"
 __author_email__ = "aborruso@gmail.com"
 

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/scrape_cli/scrape.py

@@ -6,17 +6,21 @@
 # $ curl 'https://en.wikipedia.org/wiki/List_of_sovereign_states' -s \
 # | scrape -e 'table.wikitable > tbody > tr > td > b > a'
 #
-# Dependencies: lxml, cssselect, requests
+# Dependencies: lxml, cssselect, requests, xmltodict
 #
 # Author: http://jeroenjanssens.com
 
 import os
 import sys
 import re
+import json
 import argparse
 import requests
+import xmltodict
 from lxml import etree
 from cssselect import GenericTranslator
+from xml.parsers.expat import ExpatError
+from xml.sax.saxutils import escape
 
 # Read version from __init__.py
 init_file = os.path.join(os.path.dirname(__file__), '__init__.py')
@@ -109,15 +113,19 @@ examples:
   cat file.html | scrape -e "//h1"                read from stdin
   scrape -e "//h1" -x file.html                   check existence (exit 0/1)
   scrape -be "//article" file.html                wrap output in <html><body>
+  scrape -je "//article" file.html                output as JSON (built-in)
 '''
     )
 
     parser.add_argument('--version', action='version', version=f'%(prog)s {__version__}')
 
-    # Check for incorrect argument order (-eb instead of -be)
-    if '-eb' in ' '.join(sys.argv):
+    # Check for incorrect argument order (-eb instead of -be, -ej instead of -je)
+    if '-eb' in sys.argv[1:]:
         print("Error: use -be not -eb.\n  scrape -be \"//article\" file.html", file=sys.stderr)
         sys.exit(1)
+    if '-ej' in sys.argv[1:]:
+        print("Error: use -je not -ej.\n  scrape -je \"//article\" file.html", file=sys.stderr)
+        sys.exit(1)
     # Defines the HTML input argument (can be a file, URL or stdin)
     parser.add_argument('html', nargs='?', type=str, default='',
                         help="HTML input (file, URL or stdin, default: stdin)", metavar="HTML")
@@ -130,6 +138,9 @@ examples:
     # Option to extract only text content
     parser.add_argument('-t', '--text', action='store_true', default=False,
                         help="Extract only text content (useful for LLMs)")
+    # Option to output structured JSON (built-in, no external xq needed)
+    parser.add_argument('-j', '--json', action='store_true', default=False,
+                        help="Output result as structured JSON")
     # Allows to specify one or more XPath or CSS3 selector expressions
     parser.add_argument('-e', '--expression', default=[], action='append',
                         help="XPath query or CSS3 selector")
@@ -143,6 +154,17 @@ examples:
                         help="Custom User-Agent string for HTTP requests")
     args = parser.parse_args()
 
+    # JSON flag is mutually exclusive with -t, -x, -a
+    if args.json and args.text:
+        print("Error: --json and --text are mutually exclusive.", file=sys.stderr)
+        sys.exit(1)
+    if args.json and args.check_existence:
+        print("Error: --json and --check-existence are mutually exclusive.", file=sys.stderr)
+        sys.exit(1)
+    if args.json and args.argument:
+        print("Error: --json cannot be combined with -a/--argument.", file=sys.stderr)
+        sys.exit(1)
+
     # Check that at least one expression is provided by the user (unless using -t option)
     if not args.expression and not args.text:
         print(
@@ -241,6 +263,7 @@ examples:
         sys.exit(1)
 
     results = []
+    string_results = []
     # For each expression, perform the search in the parsed HTML
     for e in expression:
         els = list(document.xpath(e))
@@ -257,6 +280,10 @@ examples:
                 # Clean the text when using -t option
                 if args.text:
                     text = clean_text(text)
+                # Strings are plain text, but -j parses the results as XML
+                if args.json:
+                    string_results.append(text.strip())
+                    text = escape(text)
             elif args.text:
                 # If -t option is used, extract only text content
                 if hasattr(el, 'text_content'):
@@ -281,7 +308,21 @@ examples:
         results = [final_text] if final_text else []
 
     # Output handling
-    if args.body and not args.text:
+    if args.json:
+        values = [s for s in string_results if s]
+        if values and len(string_results) == len(results):
+            # Only strings (@attr, text()): XML conversion would merge them into one
+            parsed = {"html": {"body": values[0] if len(values) == 1 else values}}
+        else:
+            wrapped = "<!DOCTYPE html>\n<html>\n<body>\n" + "".join(r + "\n" for r in results) + "</body>\n</html>\n"
+            try:
+                parsed = xmltodict.parse(wrapped)
+            except ExpatError as e:
+                print(f"Error: cannot convert result to JSON, it is not valid XML ({e}). "
+                      "Attribute names like @click or :bind are not supported by -j.", file=sys.stderr)
+                sys.exit(1)
+        sys.stdout.write(json.dumps(parsed, indent=2, ensure_ascii=False) + "\n")
+    elif args.body and not args.text:
         sys.stdout.write("<!DOCTYPE html>\n<html>\n<body>\n")
         for result in results:
             sys.stdout.write(result + "\n")

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/scrape_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.2.4
+Version: 1.3.1
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -11,6 +11,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
+Requires-Dist: xmltodict
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)
@@ -71,6 +72,7 @@ pip install -e .
 - requests
 - lxml
 - cssselect
+- xmltodict
 
 ## How does it work?
 
@@ -230,13 +232,12 @@ scrape -te 'h1, h2, h3' resources/test.html
 
 The `-t` option automatically excludes text from `<script>` and `<style>` tags and cleans up whitespace for better readability.
 
-### JSON Output Integration
+### JSON Output
 
-You can integrate scrape-cli with [xq](https://github.com/kislyuk/yq) (part of yq) to convert HTML output to structured JSON:
+Use the `-j`/`--json` flag to get structured JSON natively, with no external tools:
 
 ```bash
-# Extract and convert to JSON (requires -b for complete HTML)
-scrape -be "a.external-link" resources/test.html | xq .
+scrape -je "a.external-link" resources/test.html
 ```
 
 Output:
@@ -258,7 +259,7 @@ Output:
 Table extraction example:
 
 ```bash
-scrape -be "table.data-table td" resources/test.html | xq .
+scrape -je "table.data-table td" resources/test.html
 ```
 
 Output:
@@ -268,19 +269,23 @@ Output:
   "html": {
     "body": {
       "td": [
-        "1",
-        "John Doe",
-        "john@example.com",
-        "2",
-        "Jane Smith",
-        "jane@example.com"
+        "Italy",
+        "Rome",
+        "59",
+        "France",
+        "Paris",
+        "68"
       ]
     }
   }
 }
 ```
 
-**Note**: The `-b` flag is mandatory to produce valid HTML with `<html>`, `<head>` and `<body>` tags.
+`-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
+
+`-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
+
+`-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.
 
 Useful for JSON-based pipelines, APIs, databases, and processing with jq/DuckDB.
 
@@ -288,6 +293,7 @@ Some notes on the commands:
 
 - `-e` to set the query
 - `-b` to add `<html>`, `<head>` and `<body>` tags to the HTML output
+- `-j` to output structured JSON (built-in)
 - `-t` to extract only text content (useful for LLMs and text processing)
 
 

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/scrape_cli.egg-info/requires.txt

@@ -1,3 +1,4 @@
 cssselect
 lxml
 requests
+xmltodict

{scrape_cli-1.2.4 → scrape_cli-1.3.1}/tests/test_scrape.py

@@ -1,3 +1,4 @@
+import json
 import subprocess
 import sys
 import threading
@@ -182,6 +183,26 @@ def test_incorrect_eb_order_exits_with_specific_message():
     assert "use -be not -eb" in result.stderr
 
 
+def test_incorrect_ej_order_exits_with_specific_message():
+    result = run_scrape("-ej")
+
+    assert result.returncode == 1
+    assert "use -je not -ej" in result.stderr
+
+
+def test_order_checks_ignore_filenames_and_selectors(tmp_path):
+    for name in ("page-eb.html", "page-ej.html"):
+        html_file = tmp_path / name
+        html_file.write_text("<h1>ok</h1>", encoding="utf-8")
+        result = run_scrape(str(html_file), "-e", "h1")
+        assert result.returncode == 0, result.stderr
+        assert "ok" in result.stdout
+
+    for selector in (".page-ejected", ".page-ebook"):
+        result = run_scrape("-e", selector, input_data='<p class="x">ok</p>')
+        assert "use -" not in result.stderr
+
+
 def test_invalid_css_selector_fails_conversion():
     result = run_scrape(str(TEST_HTML), "-e", "div[")
 
@@ -189,6 +210,100 @@ def test_invalid_css_selector_fails_conversion():
     assert "Error converting CSS selector to XPath" in result.stderr
 
 
+def test_json_flag_produces_valid_json_for_single_element():
+    result = run_scrape(str(TEST_HTML), "-je", "a.external-link")
+
+    assert result.returncode == 0
+    payload = json.loads(result.stdout)
+    a = payload["html"]["body"]["a"]
+    assert a["@href"] == "https://example.com"
+    assert a["@class"] == "external-link"
+    assert a["#text"] == "Example Link"
+
+
+def test_json_flag_produces_array_for_multiple_elements():
+    result = run_scrape(str(TEST_HTML), "-je", "table.data-table td")
+
+    assert result.returncode == 0
+    payload = json.loads(result.stdout)
+    tds = payload["html"]["body"]["td"]
+    assert isinstance(tds, list)
+    assert "Italy" in tds
+    assert "Rome" in tds
+
+
+def test_json_flag_works_without_b_flag():
+    only_j = run_scrape(str(TEST_HTML), "-j", "-e", "a.external-link")
+    with_bj = run_scrape(str(TEST_HTML), "-bj", "-e", "a.external-link")
+
+    assert only_j.returncode == 0
+    assert with_bj.returncode == 0
+    assert only_j.stdout == with_bj.stdout
+
+
+def test_json_flag_conflicts_with_text_check_existence_argument():
+    jt = run_scrape(str(TEST_HTML), "-j", "-t")
+    assert jt.returncode == 1
+    assert "mutually exclusive" in jt.stderr
+
+    jx = run_scrape(str(TEST_HTML), "-j", "-e", "//h1", "-x")
+    assert jx.returncode == 1
+    assert "mutually exclusive" in jx.stderr
+
+    ja = run_scrape(str(TEST_HTML), "-j", "-e", "//a", "-a", "href")
+    assert ja.returncode == 1
+    assert "cannot be combined" in ja.stderr
+
+
+def test_json_flag_handles_string_results_with_special_characters():
+    html = '<a href="x?a=1&amp;b=2">AT&amp;T &lt;b&gt;</a>'
+
+    attr = run_scrape("-je", "//a/@href", input_data=html)
+    assert attr.returncode == 0, attr.stderr
+    assert json.loads(attr.stdout)["html"]["body"] == "x?a=1&b=2"
+
+    text = run_scrape("-je", "//a/text()", input_data=html)
+    assert text.returncode == 0, text.stderr
+    assert json.loads(text.stdout)["html"]["body"] == "AT&T <b>"
+
+
+def test_json_flag_returns_array_for_multiple_string_results():
+    html = '<a href="x?a=1&amp;b=2">uno</a> <a href="y">due</a>'
+
+    attrs = run_scrape("-je", "//a/@href", input_data=html)
+    assert attrs.returncode == 0, attrs.stderr
+    assert json.loads(attrs.stdout)["html"]["body"] == ["x?a=1&b=2", "y"]
+
+    # Whitespace-only text nodes are not results
+    texts = run_scrape("-je", "//body//text()", input_data=html)
+    assert json.loads(texts.stdout)["html"]["body"] == ["uno", "due"]
+
+
+def test_json_flag_mixed_elements_and_strings_still_converts():
+    result = run_scrape("-je", "//a | //a/@href", input_data='<a href="x&amp;y">uno</a>')
+
+    assert result.returncode == 0, result.stderr
+    body = json.loads(result.stdout)["html"]["body"]
+    assert body["a"]["#text"] == "uno"
+    assert body["#text"] == "x&y"
+
+
+def test_json_flag_text_results_work_on_test_file():
+    result = run_scrape(str(TEST_HTML), "-je", "//p/text()")
+
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout)["html"]["body"]
+
+
+def test_json_flag_reports_clean_error_on_non_xml_attribute_name():
+    result = run_scrape("-je", "#v", input_data='<div id="v" @click="go">x</div>')
+
+    assert result.returncode == 1
+    assert "Error:" in result.stderr
+    assert "Traceback" not in result.stderr
+    assert result.stdout == ""
+
+
 def test_url_input_downloads_and_extracts_text():
     html_bytes = TEST_HTML.read_bytes()
     server, thread = run_test_server(html_bytes)