scrape-cli 1.3.2__tar.gz → 1.3.4__tar.gz

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.2
+Version: 1.3.4
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -68,7 +68,7 @@ pip install -e .
 
 ## Requirements
 
-- Python >=3.6
+- Python >=3.8
 - requests
 - lxml
 - cssselect

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/README.md

@@ -53,7 +53,7 @@ pip install -e .
 
 ## Requirements
 
-- Python >=3.6
+- Python >=3.8
 - requests
 - lxml
 - cssselect

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrape_cli"
-version = "1.3.2"
+version = "1.3.4"
 description = "It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector."
 readme = "README.md"
 authors = [
@@ -27,6 +27,7 @@ Homepage = "https://github.com/aborruso/scrape-cli"
 
 [project.scripts]
 scrape = "scrape_cli.scrape:main"
+scrape-cli = "scrape_cli.scrape:main"
 
 [tool.setuptools.packages.find]
 where = ["."]

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/scrape_cli/__init__.py

@@ -4,7 +4,7 @@ scrape-cli - A command-line tool to extract HTML elements using XPath or CSS3 se
 
 from scrape_cli.scrape import main
 
-__version__ = "1.3.2"
+__version__ = "1.3.4"
 __author__ = "Andrea Borruso"
 __author_email__ = "aborruso@gmail.com"
 

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/scrape_cli/scrape.py

@@ -116,19 +116,54 @@ def is_xpath(expression):
 def main():
     # Command line argument parser definition
     parser = argparse.ArgumentParser(
-        description='Extract HTML elements using an XPath query or CSS3 selector.',
+        description=(
+            'Extract parts of an HTML page using an XPath query or CSS3 selector.\n'
+            'Reads static HTML and does not run JavaScript: content built in the\n'
+            'browser is not there. Never prompts for input.'
+        ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog='''\
 examples:
-  scrape -e "//h1" file.html                      XPath expression
-  scrape -e "h1.title" file.html                  CSS selector
-  scrape -e "//a" -a href file.html               extract attribute
-  scrape -t file.html                             extract all text
-  scrape -e "//h1" https://example.com            fetch from URL
-  cat file.html | scrape -e "//h1"                read from stdin
-  scrape -e "//h1" -x file.html                   check existence (exit 0/1)
-  scrape -be "//article" file.html                wrap output in <html><body>
-  scrape -je "//article" file.html                output as JSON (built-in)
+  scrape -e "//h1" file.html            XPath: matching elements as HTML
+  scrape -e "h1.title" file.html        CSS selector, detected automatically
+  scrape -e "//a/@href" file.html       attribute values as plain text
+  scrape -e "//a" -a href file.html     same, elements without href skipped
+  scrape -e "count(//a)" file.html      XPath function: one value (string,
+                                        number, true/false)
+  scrape -t file.html                   all body text, no script or style
+  scrape -te "//article" file.html      text of the matching elements
+  scrape -je "a" file.html              JSON
+  scrape -be "//article" file.html      wrap output in <html><body>
+  scrape -x -e "//h1" file.html         test only: exit code, no output
+  scrape -e "//h1" https://example.com  fetch from URL
+  cat file.html | scrape -e "//h1"      read from stdin
+
+output:
+  default  one result after the other, on stdout: elements as HTML, strings
+           (@attr, text(), XPath functions) as plain text
+  -e       repeated: results follow the order of the expressions
+  -j       always {"html": {"body": ...}}, where body is:
+             elements  attributes as @name, text as #text; one match of a tag
+                       is an object, several are a list
+             strings   one is a string, several are a list
+             nothing   null
+
+exit codes:
+  0  success, also when nothing matches: empty output, or null body with -j
+  1  error, with a message on stderr; with -x, no match, with stderr empty
+  2  invalid usage: unknown option or missing option value
+
+notes:
+  - No match is not an error: use -x to test for it.
+  - With short flags together, -e goes last: -be, -je, -te. As an argument on
+    its own, -eb or -ej is rejected, because -e would take "b" or "j" as value.
+  - -x evaluates only the first -e.
+  - -t with -e includes the text of any script or style inside the matches.
+  - -j fails with exit 1 on attribute names that are not valid XML, such as
+    @click or :bind. Use -e without -j on those pages.
+  - URLs: 30 s timeout, HTTP errors (4xx, 5xx) exit 1.
+  - Charset: from the meta tag, otherwise UTF-8, and ISO-8859-1 if the bytes
+    are not valid UTF-8. Other encodings need the meta tag.
 '''
     )
 
@@ -141,30 +176,34 @@ examples:
             sys.exit(1)
     # Defines the HTML input argument (can be a file, URL or stdin)
     parser.add_argument('html', nargs='?', type=str, default='',
-                        help="HTML input (file, URL or stdin, default: stdin)", metavar="HTML")
+                        help="HTML input: local file or http(s) URL. Stdin if omitted", metavar="HTML")
     # Defines the optional argument to extract from the tag
     parser.add_argument('-a', '--argument', default="",
-                        help="Argument to extract from the tag")
+                        help="Output the value of this attribute instead of the element. "
+                             "Elements without it are skipped")
     # Option to include the result within HTML and BODY tags
     parser.add_argument('-b', '--body', action='store_true', default=False,
-                        help="Include result in HTML and BODY tags")
+                        help="Wrap the output in a <!DOCTYPE html><html><body> document. "
+                             "Ignored with -t and -j")
     # Option to extract only text content
     parser.add_argument('-t', '--text', action='store_true', default=False,
-                        help="Extract only text content (useful for LLMs)")
+                        help="Output text only, with whitespace cleaned. "
+                             "Without -e, all body text except script and style")
     # Option to output structured JSON (built-in, no external xq needed)
     parser.add_argument('-j', '--json', action='store_true', default=False,
-                        help="Output result as structured JSON")
+                        help="Output structured JSON, see below. Not with -t, -x, -a")
     # Allows to specify one or more XPath or CSS3 selector expressions
     parser.add_argument('-e', '--expression', default=[], action='append',
-                        help="XPath query or CSS3 selector")
+                        help="XPath query or CSS3 selector, detected automatically. Repeatable")
     # Option to verify the existence of elements matching the expression
     parser.add_argument('-x', '--check-existence', dest='check_existence', action='store_true', default=False,
-                        help="Returns an exit value indicating existence")
+                        help="Output nothing, exit 0 if the first expression matches, 1 if not")
     # Option to avoid initial HTML parsing, useful in specific cases like CData
     parser.add_argument('-r', '--rawinput', action='store_true', default=False,
-                        help="Do not parse HTML before passing to etree (useful for CData)")
+                        help="Parse the input as strict XML instead of HTML (useful for CData). "
+                             "Malformed input is an error")
     parser.add_argument('-u', '--user-agent', default=None,
-                        help="Custom User-Agent string for HTTP requests")
+                        help="User-Agent for URL requests")
     args = parser.parse_args()
 
     # JSON flag is mutually exclusive with -t, -x, -a
@@ -208,6 +247,9 @@ examples:
             except FileNotFoundError:
                 print(f"Error: The file '{args.html}' was not found.", file=sys.stderr)
                 sys.exit(1)
+            except OSError as e:
+                print(f"Error: cannot read '{args.html}': {e.strerror}", file=sys.stderr)
+                sys.exit(1)
     else:
         # If the input is from stdin
         try:
@@ -257,19 +299,16 @@ examples:
                 # If detected charset fails, fall back to default behavior
                 pass
 
-        # Try UTF-8 first, fallback to ISO-8859-1 if that fails
-        try:
-            if args.rawinput:
-                document = etree.fromstring(inp)
-            else:
-                document = etree.fromstring(inp, html_parser)
-        except UnicodeDecodeError:
-            # If UTF-8 fails, try ISO-8859-1
-            inp = inp.decode('iso-8859-1').encode('utf-8')
-            if args.rawinput:
-                document = etree.fromstring(inp)
-            else:
-                document = etree.fromstring(inp, html_parser)
+        if args.rawinput:
+            # XML declares its own encoding: leave the bytes to the parser
+            document = etree.fromstring(inp)
+        else:
+            # The recovering parser never fails on bad bytes, so check them here
+            try:
+                inp.decode('utf-8')
+            except UnicodeDecodeError:
+                inp = inp.decode('iso-8859-1').encode('utf-8')
+            document = etree.fromstring(inp, html_parser)
     except (etree.XMLSyntaxError, UnicodeDecodeError) as e:
         # Print an error in case of syntax issues in the HTML
         print(f"Error parsing HTML: {e}", file=sys.stderr)
@@ -279,7 +318,11 @@ examples:
     string_results = []
     # For each expression, perform the search in the parsed HTML
     for e in expression:
-        result = document.xpath(e)
+        try:
+            result = document.xpath(e)
+        except etree.XPathError as err:
+            print(f"Error: invalid XPath expression '{e}': {err}", file=sys.stderr)
+            sys.exit(1)
 
         # If check-existence is enabled, return 0 or 1 depending on the existence of elements
         if args.check_existence:

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/scrape_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.2
+Version: 1.3.4
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -68,7 +68,7 @@ pip install -e .
 
 ## Requirements
 
-- Python >=3.6
+- Python >=3.8
 - requests
 - lxml
 - cssselect

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/scrape_cli.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 scrape = scrape_cli.scrape:main
+scrape-cli = scrape_cli.scrape:main

{scrape_cli-1.3.2 → scrape_cli-1.3.4}/tests/test_scrape.py

@@ -107,6 +107,35 @@ def test_encoding_meta_charset_iso_8859_1(tmp_path):
     assert result.stdout.strip() == "Perché"
 
 
+def test_encoding_falls_back_to_iso_8859_1_without_meta(tmp_path):
+    html_file = tmp_path / "latin1.html"
+    html_file.write_bytes("<h1>Caffè</h1>".encode("iso-8859-1"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè\n"
+
+
+def test_encoding_utf8_without_meta_is_not_converted(tmp_path):
+    html_file = tmp_path / "utf8.html"
+    html_file.write_bytes("<h1>Caffè €</h1>".encode("utf-8"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.stdout == "Caffè €\n"
+
+
+def test_rawinput_keeps_the_encoding_declared_by_the_xml(tmp_path):
+    xml_file = tmp_path / "latin1.xml"
+    xml_file.write_bytes('<?xml version="1.0" encoding="iso-8859-1"?><a>Caffè</a>'.encode("iso-8859-1"))
+
+    result = run_scrape(str(xml_file), "-r", "-e", "string(//a)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè\n"
+
+
 def test_argument_extracts_attribute_value():
     result = run_scrape(str(TEST_HTML), "-e", "//a[@class='external-link']", "-a", "href")
 
@@ -203,6 +232,46 @@ def test_order_checks_ignore_filenames_and_selectors(tmp_path):
         assert "use -" not in result.stderr
 
 
+def test_invalid_xpath_reports_clean_error():
+    result = run_scrape("-e", "//[bad", input_data="<p>x</p>")
+
+    assert result.returncode == 1
+    assert "Error: invalid XPath" in result.stderr
+    assert "Traceback" not in result.stderr
+
+
+def test_unreadable_input_reports_clean_error(tmp_path):
+    result = run_scrape("-e", "h1", str(tmp_path))
+
+    assert result.returncode == 1
+    assert "Error: cannot read" in result.stderr
+    assert "Traceback" not in result.stderr
+
+
+def test_help_documents_the_contract_for_agents():
+    result = run_scrape("--help")
+
+    assert result.returncode == 0
+    for section in ("output:", "exit codes:", "notes:", "examples:"):
+        assert section in result.stdout
+
+
+def test_exit_codes_match_help():
+    html = "<h1>x</h1>"
+
+    no_match = run_scrape("-e", "//nulla", input_data=html)
+    assert (no_match.returncode, no_match.stdout) == (0, "")
+
+    not_found = run_scrape("-x", "-e", "//nulla", input_data=html)
+    assert not_found.returncode == 1
+    assert "Error" not in not_found.stderr
+
+    assert run_scrape("--bogus", input_data=html).returncode == 2
+
+    first_only = run_scrape("-x", "-e", "//nulla", "-e", "//h1", input_data=html)
+    assert first_only.returncode == 1
+
+
 def test_invalid_css_selector_fails_conversion():
     result = run_scrape(str(TEST_HTML), "-e", "div[")