scrape-cli 1.3.3__tar.gz → 1.3.4__tar.gz

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.3
+Version: 1.3.4
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrape_cli"
-version = "1.3.3"
+version = "1.3.4"
 description = "It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector."
 readme = "README.md"
 authors = [
@@ -27,6 +27,7 @@ Homepage = "https://github.com/aborruso/scrape-cli"
 
 [project.scripts]
 scrape = "scrape_cli.scrape:main"
+scrape-cli = "scrape_cli.scrape:main"
 
 [tool.setuptools.packages.find]
 where = ["."]

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/scrape_cli/__init__.py

@@ -4,7 +4,7 @@ scrape-cli - A command-line tool to extract HTML elements using XPath or CSS3 se
 
 from scrape_cli.scrape import main
 
-__version__ = "1.3.3"
+__version__ = "1.3.4"
 __author__ = "Andrea Borruso"
 __author_email__ = "aborruso@gmail.com"
 

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/scrape_cli/scrape.py

@@ -162,8 +162,8 @@ notes:
   - -j fails with exit 1 on attribute names that are not valid XML, such as
     @click or :bind. Use -e without -j on those pages.
   - URLs: 30 s timeout, HTTP errors (4xx, 5xx) exit 1.
-  - Charset: from the meta tag, otherwise UTF-8. Other encodings need the meta
-    tag, or characters come out wrong.
+  - Charset: from the meta tag, otherwise UTF-8, and ISO-8859-1 if the bytes
+    are not valid UTF-8. Other encodings need the meta tag.
 '''
     )
 
@@ -299,19 +299,16 @@ notes:
                 # If detected charset fails, fall back to default behavior
                 pass
 
-        # Try UTF-8 first, fallback to ISO-8859-1 if that fails
-        try:
-            if args.rawinput:
-                document = etree.fromstring(inp)
-            else:
-                document = etree.fromstring(inp, html_parser)
-        except UnicodeDecodeError:
-            # If UTF-8 fails, try ISO-8859-1
-            inp = inp.decode('iso-8859-1').encode('utf-8')
-            if args.rawinput:
-                document = etree.fromstring(inp)
-            else:
-                document = etree.fromstring(inp, html_parser)
+        if args.rawinput:
+            # XML declares its own encoding: leave the bytes to the parser
+            document = etree.fromstring(inp)
+        else:
+            # The recovering parser never fails on bad bytes, so check them here
+            try:
+                inp.decode('utf-8')
+            except UnicodeDecodeError:
+                inp = inp.decode('iso-8859-1').encode('utf-8')
+            document = etree.fromstring(inp, html_parser)
     except (etree.XMLSyntaxError, UnicodeDecodeError) as e:
         # Print an error in case of syntax issues in the HTML
         print(f"Error parsing HTML: {e}", file=sys.stderr)

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/scrape_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.3
+Version: 1.3.4
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/scrape_cli.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 scrape = scrape_cli.scrape:main
+scrape-cli = scrape_cli.scrape:main

{scrape_cli-1.3.3 → scrape_cli-1.3.4}/tests/test_scrape.py

@@ -107,6 +107,35 @@ def test_encoding_meta_charset_iso_8859_1(tmp_path):
     assert result.stdout.strip() == "Perché"
 
 
+def test_encoding_falls_back_to_iso_8859_1_without_meta(tmp_path):
+    html_file = tmp_path / "latin1.html"
+    html_file.write_bytes("<h1>Caffè</h1>".encode("iso-8859-1"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè\n"
+
+
+def test_encoding_utf8_without_meta_is_not_converted(tmp_path):
+    html_file = tmp_path / "utf8.html"
+    html_file.write_bytes("<h1>Caffè €</h1>".encode("utf-8"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.stdout == "Caffè €\n"
+
+
+def test_rawinput_keeps_the_encoding_declared_by_the_xml(tmp_path):
+    xml_file = tmp_path / "latin1.xml"
+    xml_file.write_bytes('<?xml version="1.0" encoding="iso-8859-1"?><a>Caffè</a>'.encode("iso-8859-1"))
+
+    result = run_scrape(str(xml_file), "-r", "-e", "string(//a)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè\n"
+
+
 def test_argument_extracts_attribute_value():
     result = run_scrape(str(TEST_HTML), "-e", "//a[@class='external-link']", "-a", "href")