scrape-cli 1.3.4__tar.gz → 1.3.5__tar.gz

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.4
+Version: 1.3.5
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -12,6 +12,8 @@ Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
 Requires-Dist: xmltodict
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrape_cli"
-version = "1.3.4"
+version = "1.3.5"
 description = "It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector."
 readme = "README.md"
 authors = [
@@ -22,6 +22,9 @@ dependencies = [
     "xmltodict"
 ]
 
+[project.optional-dependencies]
+dev = ["pytest"]
+
 [project.urls]
 Homepage = "https://github.com/aborruso/scrape-cli"
 

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/scrape_cli/__init__.py

@@ -4,7 +4,7 @@ scrape-cli - A command-line tool to extract HTML elements using XPath or CSS3 se
 
 from scrape_cli.scrape import main
 
-__version__ = "1.3.4"
+__version__ = "1.3.5"
 __author__ = "Andrea Borruso"
 __author_email__ = "aborruso@gmail.com"
 

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/scrape_cli/scrape.py

@@ -162,8 +162,8 @@ notes:
   - -j fails with exit 1 on attribute names that are not valid XML, such as
     @click or :bind. Use -e without -j on those pages.
   - URLs: 30 s timeout, HTTP errors (4xx, 5xx) exit 1.
-  - Charset: from the meta tag, otherwise UTF-8, and ISO-8859-1 if the bytes
-    are not valid UTF-8. Other encodings need the meta tag.
+  - Charset: from the meta tag, otherwise UTF-8. If the bytes are not valid
+    UTF-8: Windows-1252, then ISO-8859-1. Other encodings need the meta tag.
 '''
     )
 
@@ -307,7 +307,11 @@ notes:
             try:
                 inp.decode('utf-8')
             except UnicodeDecodeError:
-                inp = inp.decode('iso-8859-1').encode('utf-8')
+                # cp1252 rejects five undefined bytes, iso-8859-1 accepts them all
+                try:
+                    inp = inp.decode('cp1252').encode('utf-8')
+                except UnicodeDecodeError:
+                    inp = inp.decode('iso-8859-1').encode('utf-8')
             document = etree.fromstring(inp, html_parser)
     except (etree.XMLSyntaxError, UnicodeDecodeError) as e:
         # Print an error in case of syntax issues in the HTML

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/scrape_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrape_cli
-Version: 1.3.4
+Version: 1.3.5
 Summary: It's a command-line tool to extract HTML elements using an XPath query or CSS3 selector.
 Author-email: Andrea Borruso <aborruso@gmail.com>
 Project-URL: Homepage, https://github.com/aborruso/scrape-cli
@@ -12,6 +12,8 @@ Requires-Dist: cssselect
 Requires-Dist: lxml
 Requires-Dist: requests
 Requires-Dist: xmltodict
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 
 [![PyPI version](https://img.shields.io/pypi/v/scrape-cli.svg?label=PyPI%20version)](https://pypi.org/project/scrape-cli/)
 [![Python Versions](https://img.shields.io/pypi/pyversions/scrape-cli.svg)](https://pypi.org/project/scrape-cli/)

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/scrape_cli.egg-info/requires.txt

@@ -2,3 +2,6 @@ cssselect
 lxml
 requests
 xmltodict
+
+[dev]
+pytest

{scrape_cli-1.3.4 → scrape_cli-1.3.5}/tests/test_scrape.py

@@ -117,6 +117,26 @@ def test_encoding_falls_back_to_iso_8859_1_without_meta(tmp_path):
     assert result.stdout == "Caffè\n"
 
 
+def test_encoding_windows_1252_without_meta(tmp_path):
+    html_file = tmp_path / "cp1252.html"
+    html_file.write_bytes("<h1>Caffè € ’</h1>".encode("cp1252"))
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout == "Caffè € ’\n"
+
+
+def test_encoding_bytes_undefined_in_windows_1252_fall_back_to_iso_8859_1(tmp_path):
+    html_file = tmp_path / "undefined.html"
+    html_file.write_bytes(b"<h1>Caff\xe8 \x81</h1>")
+
+    result = run_scrape(str(html_file), "-e", "string(//h1)")
+
+    assert result.returncode == 0
+    assert result.stdout.startswith("Caffè")
+
+
 def test_encoding_utf8_without_meta_is_not_converted(tmp_path):
     html_file = tmp_path / "utf8.html"
     html_file.write_bytes("<h1>Caffè €</h1>".encode("utf-8"))