scrape-cli 1.3.4__tar.gz → 1.3.6__tar.gz

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.4
+Version: 1.3.6
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -12,6 +12,8 @@ Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
 Requires-Dist: xmltodict
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)
@@ -283,6 +285,8 @@ Output:
 
 `-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
 
+Use `--json-array` when a script needs one shape for any number of results. The output is a flat list with no `html`/`body` envelope, one item per result, in the order of the expressions. Strings stay strings and elements become objects with the tag as key, so `scrape --json-array -e "//a/@href" -e "b"` gives `["x", "y", {"b": "tre"}]`, and no results give `[]`. It implies `-j` and has the same limits.
+
 `-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
 
 `-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/README.md

@@ -268,6 +268,8 @@ Output:
 
 `-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
 
+Use `--json-array` when a script needs one shape for any number of results. The output is a flat list with no `html`/`body` envelope, one item per result, in the order of the expressions. Strings stay strings and elements become objects with the tag as key, so `scrape --json-array -e "//a/@href" -e "b"` gives `["x", "y", {"b": "tre"}]`, and no results give `[]`. It implies `-j` and has the same limits.
+
 `-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
 
 `-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrape_cli"
-version = "1.3.4"
+version = "1.3.6"
 description = "It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector."
 readme = "README.md"
 authors = [
@@ -22,6 +22,9 @@ dependencies = [
     "xmltodict"
 ]
 
+[project.optional-dependencies]
+dev = ["pytest"]
+
 [project.urls]
 Homepage = "https://github.com/aborruso/scrape-cli"
 

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/scrape_cli/__init__.py

@@ -4,7 +4,7 @@ scrape-cli - A command-line tool to extract HTML elements using XPath or CSS3 se
 
 from scrape_cli.scrape import main
 
-__version__ = "1.3.4"
+__version__ = "1.3.6"
 __author__ = "Andrea Borruso"
 __author_email__ = "aborruso@gmail.com"
 

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/scrape_cli/scrape.py

@@ -133,6 +133,7 @@ examples:
   scrape -t file.html                   all body text, no script or style
   scrape -te "//article" file.html      text of the matching elements
   scrape -je "a" file.html              JSON
+  scrape --json-array -e "a" file.html  JSON, always a flat list
   scrape -be "//article" file.html      wrap output in <html><body>
   scrape -x -e "//h1" file.html         test only: exit code, no output
   scrape -e "//h1" https://example.com  fetch from URL
@@ -147,6 +148,8 @@ output:
                        is an object, several are a list
              strings   one is a string, several are a list
              nothing   null
+  --json-array  always a flat list, one item per result, in expression order:
+             [] for nothing, strings as strings, elements as {"tag": ...}
 
 exit codes:
   0  success, also when nothing matches: empty output, or null body with -j
@@ -155,6 +158,7 @@ exit codes:
 
 notes:
   - No match is not an error: use -x to test for it.
+  - HTML with no elements, such as only spaces or comments, exits 1.
   - With short flags together, -e goes last: -be, -je, -te. As an argument on
     its own, -eb or -ej is rejected, because -e would take "b" or "j" as value.
   - -x evaluates only the first -e.
@@ -162,8 +166,8 @@ notes:
   - -j fails with exit 1 on attribute names that are not valid XML, such as
     @click or :bind. Use -e without -j on those pages.
   - URLs: 30 s timeout, HTTP errors (4xx, 5xx) exit 1.
-  - Charset: from the meta tag, otherwise UTF-8, and ISO-8859-1 if the bytes
-    are not valid UTF-8. Other encodings need the meta tag.
+  - Charset: from the meta tag, otherwise UTF-8. If the bytes are not valid
+    UTF-8: Windows-1252, then ISO-8859-1. Other encodings need the meta tag.
 '''
     )
 
@@ -192,6 +196,8 @@ notes:
     # Option to output structured JSON (built-in, no external xq needed)
     parser.add_argument('-j', '--json', action='store_true', default=False,
                         help="Output structured JSON, see below. Not with -t, -x, -a")
+    parser.add_argument('--json-array', action='store_true', default=False,
+                        help="Output JSON as a flat list, one item per result, see below. Same limits as -j")
     # Allows to specify one or more XPath or CSS3 selector expressions
     parser.add_argument('-e', '--expression', default=[], action='append',
                         help="XPath query or CSS3 selector, detected automatically. Repeatable")
@@ -207,6 +213,8 @@ notes:
     args = parser.parse_args()
 
     # JSON flag is mutually exclusive with -t, -x, -a
+    if args.json_array:
+        args.json = True
     if args.json and args.text:
         print("Error: --json and --text are mutually exclusive.", file=sys.stderr)
         sys.exit(1)
@@ -307,15 +315,25 @@ notes:
             try:
                 inp.decode('utf-8')
             except UnicodeDecodeError:
-                inp = inp.decode('iso-8859-1').encode('utf-8')
+                # cp1252 rejects five undefined bytes, iso-8859-1 accepts them all
+                try:
+                    inp = inp.decode('cp1252').encode('utf-8')
+                except UnicodeDecodeError:
+                    inp = inp.decode('iso-8859-1').encode('utf-8')
             document = etree.fromstring(inp, html_parser)
     except (etree.XMLSyntaxError, UnicodeDecodeError) as e:
-        # Print an error in case of syntax issues in the HTML
-        print(f"Error parsing HTML: {e}", file=sys.stderr)
+        # Print an error in case of syntax issues in the input
+        print(f"Error parsing {'XML' if args.rawinput else 'HTML'}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if document is None:
+        print("Error parsing HTML: no elements found in the input", file=sys.stderr)
         sys.exit(1)
 
     results = []
     string_results = []
+    # --json-array items: str for strings, bytes for element markup
+    array_items = []
     # For each expression, perform the search in the parsed HTML
     for e in expression:
         try:
@@ -332,6 +350,13 @@ notes:
 
         # Extract the text or content of the found elements
         for el in els:
+            if args.json_array:
+                if isinstance(el, str):
+                    if el.strip():
+                        array_items.append(el.strip())
+                elif isinstance(el.tag, str):  # not a comment or processing instruction
+                    array_items.append(etree.tostring(el, with_tail=False))
+                continue
             if isinstance(el, str):
                 # If the element is a string, use the text directly
                 text = el
@@ -352,7 +377,7 @@ notes:
                 text = clean_text(text)
             elif not args.argument:
                 # If no attribute is specified, return the element as HTML string
-                text = etree.tostring(el, pretty_print=True).decode('utf-8')
+                text = etree.tostring(el, pretty_print=True, with_tail=False).decode('utf-8')
             else:
                 # Otherwise, extract the specified attribute
                 text = el.get(args.argument)
@@ -368,16 +393,18 @@ notes:
     # Output handling
     if args.json:
         values = [s for s in string_results if s]
-        if values and len(string_results) == len(results):
-            # Only strings (@attr, text()): XML conversion would merge them into one
-            parsed = {"html": {"body": values[0] if len(values) == 1 else values}}
-        else:
-            try:
+        try:
+            if args.json_array:
+                parsed = [xmltodict.parse(i) if isinstance(i, bytes) else i for i in array_items]
+            elif values and len(string_results) == len(results):
+                # Only strings (@attr, text()): XML conversion would merge them into one
+                parsed = {"html": {"body": values[0] if len(values) == 1 else values}}
+            else:
                 parsed = xmltodict.parse(wrap_in_body(results))
-            except ExpatError as e:
-                print(f"Error: cannot convert result to JSON, it is not valid XML ({e}). "
-                      "Attribute names like @click or :bind are not supported by -j.", file=sys.stderr)
-                sys.exit(1)
+        except ExpatError as e:
+            print(f"Error: cannot convert result to JSON, it is not valid XML ({e}). "
+                  "Attribute names like @click or :bind are not supported by -j.", file=sys.stderr)
+            sys.exit(1)
         sys.stdout.write(json.dumps(parsed, indent=2, ensure_ascii=False) + "\n")
     elif args.body and not args.text:
         sys.stdout.write(wrap_in_body(results))

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/scrape_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.4
+Version: 1.3.6
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -12,6 +12,8 @@ Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
 Requires-Dist: xmltodict
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)
@@ -283,6 +285,8 @@ Output:
 
 `-j` automatically wraps the result in `<html>/<body>` before conversion, so you don't need to pass `-b`. For elements, the output is the same you would get from `scrape -be ... | xq .` (the underlying converter is `xmltodict`, the same library used by `xq`). Expressions that return only strings, such as `//a/@href` or `//p/text()`, give the string in `body` when there is one result and an array when there are more.
 
+Use `--json-array` when a script needs one shape for any number of results. The output is a flat list with no `html`/`body` envelope, one item per result, in the order of the expressions. Strings stay strings and elements become objects with the tag as key, so `scrape --json-array -e "//a/@href" -e "b"` gives `["x", "y", {"b": "tre"}]`, and no results give `[]`. It implies `-j` and has the same limits.
+
 `-j` is mutually exclusive with `-t`, `-x` (`--check-existence`) and `-a` (`--argument`).
 
 `-j` exits with an error when a matched element has an attribute name that is not valid XML, such as `@click` or `:bind`.

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/scrape_cli.egg-info/requires.txt

@@ -2,3 +2,6 @@ cssselect
 lxml
 requests
 xmltodict
+
+[dev]
+pytest

{scrape_cli-1.3.4 → scrape_cli-1.3.6}/tests/test_scrape.py

@@ -117,6 +117,26 @@ def test_encoding_falls_back_to_iso_8859_1_without_meta(tmp_path):
     assert result.stdout == "Caffè\n"
 
 
+def test_encoding_windows_1252_without_meta(tmp_path):
+    html_file = tmp_path / "cp1252.html"
+    html_file.write_bytes("<h1>Caffè € ’</h1>".encode("cp1252"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè € ’\n"
+
+
+def test_encoding_bytes_undefined_in_windows_1252_fall_back_to_iso_8859_1(tmp_path):
+    html_file = tmp_path / "undefined.html"
+    html_file.write_bytes(b"<h1>Caff\xe8 \x81</h1>")
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout.startswith("Caffè")
+
+
 def test_encoding_utf8_without_meta_is_not_converted(tmp_path):
     html_file = tmp_path / "utf8.html"
     html_file.write_bytes("<h1>Caffè €</h1>".encode("utf-8"))
@@ -168,6 +188,12 @@ def test_short_check_existence_flag_x():
     assert missing.returncode == 1
 
 
+def test_rawinput_malformed_xml_reports_xml_error():
+    result = run_scrape("-r", "-e", "//a", input_data="<root><a></root>")
+    assert result.returncode == 1
+    assert "Error parsing XML: " in result.stderr
+
+
 def test_rawinput_parses_xml_without_html_parser():
     xml_data = "<root><item>one</item><item>two</item></root>"
     result = run_scrape("-e", "//item[2]/text()", "-r", input_data=xml_data)
@@ -252,7 +278,7 @@ def test_help_documents_the_contract_for_agents():
     result = run_scrape("--help")
 
     assert result.returncode == 0
-    for section in ("output:", "exit codes:", "notes:", "examples:"):
+    for section in ("output:", "exit codes:", "notes:", "examples:", "--json-array"):
         assert section in result.stdout
 
 
@@ -424,3 +450,97 @@ def test_url_input_downloads_and_extracts_text():
 
     assert result.returncode == 0
     assert result.stdout.strip() == "Welcome to the Test Page"
+
+
+TAIL_HTML = '<p>Prima <a href="/x">Link</a> dopo.</p>'
+ARRAY_HTML = (
+    '<html><body><!-- c --><p class="a">uno</p><p class="a">due</p>'
+    '<a href="x">l</a><a href="y">m</a><b>tre</b></body></html>'
+)
+
+
+def test_text_after_element_stays_out_of_html_output():
+    result = run_scrape("-e", "//a", input_data=TAIL_HTML)
+    assert result.stdout.strip() == '<a href="/x">Link</a>'
+
+
+def test_text_after_element_stays_out_of_body_output():
+    result = run_scrape("-be", "//a", input_data=TAIL_HTML)
+    assert "dopo" not in result.stdout
+
+
+def test_text_after_element_stays_out_of_json_output():
+    result = run_scrape("-je", "//a", input_data=TAIL_HTML)
+    assert json.loads(result.stdout) == {"html": {"body": {"a": {"@href": "/x", "#text": "Link"}}}}
+
+
+def json_array(*expressions, input_data=ARRAY_HTML):
+    args = ["--json-array"]
+    for expression in expressions:
+        args += ["-e", expression]
+    result = run_scrape(*args, input_data=input_data)
+    assert result.returncode == 0, result.stderr
+    return json.loads(result.stdout)
+
+
+def test_json_array_no_results():
+    assert json_array("p.none") == []
+
+
+def test_json_array_one_result():
+    assert json_array("b") == [{"b": "tre"}]
+
+
+def test_json_array_several_results():
+    assert json_array("//a/@href") == ["x", "y"]
+    assert json_array("p") == [
+        {"p": {"@class": "a", "#text": "uno"}},
+        {"p": {"@class": "a", "#text": "due"}},
+    ]
+
+
+def test_json_array_keeps_the_order_of_the_expressions():
+    assert json_array("//a/@href", "b") == ["x", "y", {"b": "tre"}]
+    assert json_array("b", "//a/@href") == [{"b": "tre"}, "x", "y"]
+
+
+def test_json_array_element_item():
+    assert json_array("a")[0] == {"a": {"@href": "x", "#text": "l"}}
+
+
+def test_json_array_leaves_out_text_after_element():
+    assert json_array("b", input_data="<p>a <b>x</b> coda</p>") == [{"b": "x"}]
+
+
+def test_json_array_xpath_functions():
+    assert json_array("count(//p)") == ["2"]
+    assert json_array("string(//none)") == []
+
+
+def test_json_array_skips_comments():
+    assert json_array("//comment()") == []
+
+
+def test_json_array_rejects_conflicting_flags():
+    for flags in (["-t"], ["-x"], ["-a", "href"]):
+        result = run_scrape("--json-array", *flags, "-e", "a", input_data=ARRAY_HTML)
+        assert result.returncode == 1
+        assert "Error:" in result.stderr
+
+
+def test_json_array_reports_clean_error_on_non_xml_attribute_name():
+    result = run_scrape("--json-array", "-e", "a", input_data='<a @click="x">l</a>')
+    assert result.returncode == 1
+    assert result.stdout == ""
+    assert "Error:" in result.stderr
+    assert "Traceback" not in result.stderr
+
+
+def test_html_input_with_no_elements_reports_clean_error():
+    for html in (" ", "<!-- c -->", '<?xml version="1.0"?>'):
+        for flags in (["-e", "//a"], ["-t"], ["-j", "-e", "//a"], ["--json-array", "-e", "//a"]):
+            result = run_scrape(*flags, input_data=html)
+            assert result.returncode == 1
+            assert result.stdout == ""
+            assert "Error parsing HTML: no elements found" in result.stderr
+            assert "Traceback" not in result.stderr