webskrap 3.2.2__tar.gz → 3.2.3__tar.gz

{webskrap-3.2.2 → webskrap-3.2.3}/CHANGELOG.md

@@ -9,6 +9,25 @@ history, so they summarize each release rather than list every change.
 
 ## [Unreleased]
 
+## [3.2.3] - 2026-10-30
+
+### Security
+
+- MCP URL checks now decode host escapes and treat backslashes as path
+  separators before checking for private addresses, as Chromium does.
+- Browser installation rejects ZIP entries with Windows separators or drive
+  components that could write outside the install folder.
+
+### Fixed
+
+- On macOS, Chrome for Testing no longer asks for the "Chromium Safe Storage"
+  keychain password at each launch. Launches now use `--use-mock-keychain`.
+  Upgrade note: on macOS, persistent sessions and `user_data_dir` profiles
+  made before this release used the real Keychain key. They lose their saved
+  cookies and logins once, and you must sign in again.
+- Search skips malformed result URLs instead of discarding all results from
+  the page. This also applies to URLs decoded from search-engine redirects.
+
 ## [3.2.2] - 2026-10-30
 
 ### Fixed
@@ -961,7 +980,8 @@ entries below.
 
 Earlier releases (0.1.0 - 0.4.9) are recorded in the git history and tags only.
 
-[Unreleased]: https://github.com/kacigaya/webskrap/compare/v3.2.2...HEAD
+[Unreleased]: https://github.com/kacigaya/webskrap/compare/v3.2.3...HEAD
+[3.2.3]: https://github.com/kacigaya/webskrap/compare/v3.2.2...v3.2.3
 [3.2.2]: https://github.com/kacigaya/webskrap/compare/v3.2.1...v3.2.2
 [3.2.1]: https://github.com/kacigaya/webskrap/compare/v3.2.0...v3.2.1
 [3.2.0]: https://github.com/kacigaya/webskrap/compare/v3.1.0...v3.2.0

{webskrap-3.2.2 → webskrap-3.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: webskrap
-Version: 3.2.2
+Version: 3.2.3
 Summary: Async Python scraping on Chromium over its own CDP client, with coherent browser profiles and session controls.
 Project-URL: Homepage, https://github.com/kacigaya/webskrap
 Project-URL: Repository, https://github.com/kacigaya/webskrap

{webskrap-3.2.2 → webskrap-3.2.3}/SECURITY.md

@@ -65,6 +65,11 @@ to permit them. The Python API and CLI take URLs from the operator and skip
 the DNS check; point them at internal hosts deliberately, not with
 page-derived input.
 
+This check applies to the supplied URL. It does not filter redirects,
+subresources, or requests from page scripts. DNS can also change after the
+check. Use network-level restrictions when the browser must not reach private
+services. The URL check alone does not provide full SSRF isolation.
+
 ### Chromium sandbox
 
 Persistent sessions keep Chromium's OS sandbox. That sandbox is what stops a

{webskrap-3.2.2 → webskrap-3.2.3}/docs/diagnostics/datadome.md

@@ -66,6 +66,30 @@ A temporary headed browser session and loopback VNC listener on port 15900 were
 to prepare manual completion. Both were closed when manual completion was
 declined. Xvfb alone does not give remote access.
 
+### Reuse a solved session
+
+A persistent session keeps its cookies in its profile on disk
+(`~/.webskrap/browser/<session>/user-data`). The DataDome cookie from a
+manual solve stays there. No extra flag or export step is necessary.
+
+1. Open the session by the same name:
+   `webskrap browser open --session <name> --headed <url>`.
+2. Keep the same exit IP. The cookie belongs to the session that passed
+   the check.
+3. Do not use the profile headless. A headless visit can receive a new
+   challenge or a hard block, and DataDome can then replace the cookie.
+
+On 2026-10-30, a macOS run checked this on a DataDome website from a
+residential IP. A person solved the CAPTCHA once in a headed session. A
+reload then returned HTTP 200. The session was closed, and a new browser
+process opened the same profile. A direct load of the protected page
+returned HTTP 200 again, without a challenge. This run did not measure how
+long the cookie stays valid. DataDome can challenge the session again at
+any time.
+
+WebSkrap does not export the cookie or copy it into other browsers. That is
+token reuse, which is out of scope.
+
 ## Verification limits
 
 The final Linux ARM64 run passed 858 tests. It excluded 30 opt-in live tests

{webskrap-3.2.2 → webskrap-3.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "webskrap"
-version = "3.2.2"
+version = "3.2.3"
 description = "Async Python scraping on Chromium over its own CDP client, with coherent browser profiles and session controls."
 readme = "README.md"
 requires-python = ">=3.11"

{webskrap-3.2.2 → webskrap-3.2.3}/src/webskrap/browser_session.py

@@ -499,6 +499,10 @@ def launch_browser(
         f"--user-data-dir={user_data_dir}",
         "--no-first-run",
         "--no-default-browser-check",
+        # Chrome for Testing is unsigned on macOS, so every launch asks the
+        # user for the "Chromium Safe Storage" keychain item. The mock keychain
+        # keeps cookie encryption profile-local; pages cannot see it.
+        *(["--use-mock-keychain"] if sys.platform == "darwin" else []),
         # A bfcache restore re-fires no load events, stranding the
         # go_back/go_forward load wait. Each command attaches anew and cannot
         # track restores across attachments, so trade bfcache for

{webskrap-3.2.2 → webskrap-3.2.3}/src/webskrap/cdp.py

@@ -861,6 +861,10 @@ async def launch(
         f"--user-data-dir={profile}",
         "--no-first-run",
         "--no-default-browser-check",
+        # Chrome for Testing is unsigned on macOS, so every launch asks the
+        # user for the "Chromium Safe Storage" keychain item. The mock keychain
+        # keeps cookie encryption profile-local; pages cannot see it.
+        *(["--use-mock-keychain"] if sys.platform == "darwin" else []),
         # Chromium builds (not branded Chrome) apply their testing field
         # trials by default. Stable Chrome never runs that config, and one of
         # its trials trims navigator.languages to a single entry.

{webskrap-3.2.2 → webskrap-3.2.3}/src/webskrap/install.py

@@ -164,7 +164,12 @@ def _download(url: str, target: IO[bytes]) -> int:
 def _safe_member(name: str) -> PurePosixPath:
     """Return ``name`` as a relative path, refusing absolute or ``..`` paths."""
     path = PurePosixPath(name)
-    if path.is_absolute() or ".." in path.parts or (path.parts and ":" in path.parts[0]):
+    if (
+        path.is_absolute()
+        or ".." in path.parts
+        or "\\" in name
+        or any(":" in part for part in path.parts)
+    ):
         msg = f"refusing archive entry outside the install folder: {name}"
         raise WebSkrapError(msg, code=ErrorCode.BROWSER_LAUNCH)
     return path

{webskrap-3.2.2 → webskrap-3.2.3}/src/webskrap/search.py

@@ -165,7 +165,10 @@ def _ddg_unwrap(href: str) -> str:
     """
     if href.startswith("//"):
         href = "https:" + href
-    parts = urlsplit(href)
+    try:
+        parts = urlsplit(href)
+    except ValueError:
+        return href
     if parts.netloc.endswith("duckduckgo.com") and parts.path.startswith("/l/"):
         return parse_qs(parts.query).get("uddg", [""])[0]
     return href
@@ -200,7 +203,10 @@ def _bing_unwrap(href: str) -> str:
     URL-safe base64 of the destination, without padding. Anything that does
     not decode comes back as given rather than failing the whole page.
     """
-    parts = urlsplit(href)
+    try:
+        parts = urlsplit(href)
+    except ValueError:
+        return href
     if not (parts.netloc.endswith("bing.com") and parts.path == "/ck/a"):
         return href
     encoded = parse_qs(parts.query).get("u", [""])[0]
@@ -292,7 +298,11 @@ def parse_results(engine: SearchEngine, html: str) -> list[SearchHit]:
     seen: set[str] = set()
     hits: list[SearchHit] = []
     for hit in spec.hits(root):
-        if urlsplit(hit.url).scheme not in ("http", "https") or hit.url in seen:
+        try:
+            scheme = urlsplit(hit.url).scheme
+        except ValueError:
+            continue
+        if scheme not in ("http", "https") or hit.url in seen:
             continue
         seen.add(hit.url)
         hits.append(hit)

{webskrap-3.2.2 → webskrap-3.2.3}/src/webskrap/urls.py

@@ -19,7 +19,7 @@ import asyncio
 import ipaddress
 import os
 import socket
-from urllib.parse import urlsplit
+from urllib.parse import unquote, urlsplit
 
 from webskrap.errors import ErrorCode, WebSkrapError
 
@@ -122,7 +122,9 @@ async def reject_private_network(url: str) -> None:
     if urlsplit(url).scheme.lower() not in ("http", "https"):
         # data: and about:blank perform no network request; nothing to guard.
         return
-    hostname = urlsplit(url).hostname or ""
+    # Chromium treats backslashes as path separators and decodes host escapes.
+    # Check the host it will contact, not urllib's literal authority.
+    hostname = unquote(urlsplit(url.replace("\\", "/")).hostname or "")
     if _host_is_blocked_literal(hostname):
         blocked = True
     else:

{webskrap-3.2.2 → webskrap-3.2.3}/tests/test_browser_session_unit.py

@@ -5,6 +5,7 @@ import os
 import signal
 import stat
 import subprocess
+import sys
 import threading
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -62,6 +63,20 @@ def _launch(tmp_path: Path, **kwargs: Any) -> None:
         browser_session.launch_browser(tmp_path, executable="/bin/chromium", **kwargs)
 
 
+def test_launch_uses_mock_keychain_on_macos_only(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    commands = _capture_launch(monkeypatch)
+
+    monkeypatch.setattr(sys, "platform", "darwin")
+    _launch(tmp_path, headless=True)
+    monkeypatch.setattr(sys, "platform", "linux")
+    _launch(tmp_path, headless=True)
+
+    assert "--use-mock-keychain" in commands[0]
+    assert "--use-mock-keychain" not in commands[1]
+
+
 def test_launch_sandboxes_by_default(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     commands = _capture_launch(monkeypatch)
 

{webskrap-3.2.2 → webskrap-3.2.3}/tests/test_install.py

@@ -183,3 +183,15 @@ def test_install_refuses_an_archive_without_the_browser(
 def test_browsers_dir_honors_the_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("WEBSKRAP_BROWSERS_PATH", str(tmp_path))
     assert install.browsers_dir() == tmp_path
+
+
+@pytest.mark.parametrize(
+    "name",
+    [r"..\escape", r"\escape", r"\\server\share\escape", "folder/C:/escape", "folder/C:escape"],
+)
+def test_extract_refuses_windows_path_escapes(tmp_path: Path, name: str) -> None:
+    archive = _archive(tmp_path, [(name, b"x", stat.S_IFREG | 0o644)])
+    with pytest.raises(WebSkrapError) as error:
+        install.extract(archive, tmp_path / "out")
+    assert error.value.code is ErrorCode.BROWSER_LAUNCH
+    assert not (tmp_path / "out").exists()

{webskrap-3.2.2 → webskrap-3.2.3}/tests/test_mcp.py

@@ -734,6 +734,8 @@ def test_guide_documents_every_error_code() -> None:
         pytest.param("ftp://example.test/x", id="ftp"),
         pytest.param("https://user:pass@example.test/", id="userinfo"),
         pytest.param("http://169.254.169.254/", id="link-local"),
+        pytest.param("http://%31%32%37.0.0.1/", id="encoded-loopback"),
+        pytest.param(r"http://127.0.0.1\example.com/", id="backslash-host"),
     ],
 )
 def test_fetch_rejects_untrusted_targets_before_launch(monkeypatch: Any, url: str) -> None:

{webskrap-3.2.2 → webskrap-3.2.3}/tests/test_search.py

@@ -7,7 +7,9 @@ engine changes its markup, refresh the fixture and fix the one extractor.
 
 from __future__ import annotations
 
+import base64
 from pathlib import Path
+from urllib.parse import quote
 
 import pytest
 
@@ -241,3 +243,25 @@ def test_unclosed_and_void_tags_do_not_swallow_later_results() -> None:
     assert [hit.url for hit in hits] == ["https://a.example/", "https://b.example/"]
     assert hits[0].title == "A one"
     assert hits[0].snippet == "first"
+
+
+@pytest.mark.parametrize("engine", list(SearchEngine))
+@pytest.mark.parametrize("wrapped", [False, True])
+def test_malformed_result_url_keeps_valid_neighbors(engine: SearchEngine, wrapped: bool) -> None:
+    href = "http://[broken"
+    if wrapped:
+        if engine is SearchEngine.DDG:
+            href = "https://duckduckgo.com/l/?uddg=" + quote(href, safe="")
+        else:
+            payload = base64.urlsafe_b64encode(href.encode()).decode().rstrip("=")
+            href = "https://www.bing.com/ck/a?u=a1" + payload
+    if engine is SearchEngine.DDG:
+        template = '<div class="result"><a class="result__a" href="{}">hit</a></div>'
+    else:
+        template = '<li class="b_algo"><h2><a href="{}">hit</a></h2></li>'
+    html = '<ol id="b_results">' + template.format(href) + template.format("https://example.com/")
+    html += "</ol>"
+
+    assert parse_results(engine, html) == [
+        SearchHit(title="hit", url="https://example.com/", snippet="")
+    ]

{webskrap-3.2.2 → webskrap-3.2.3}/tests/test_urls.py

@@ -75,6 +75,10 @@ def test_private_net_opt_in_values(monkeypatch: pytest.MonkeyPatch, value: str)
     "url",
     [
         pytest.param("http://127.0.0.1:8000/", id="loopback-v4"),
+        pytest.param("http://%31%32%37.0.0.1/", id="encoded-loopback"),
+        pytest.param("http://%6cocalhost/", id="encoded-localhost"),
+        pytest.param(r"http://127.0.0.1\example.com/", id="backslash-host"),
+        pytest.param(r"http://%31%32%37.0.0.1\example.com/", id="encoded-backslash-host"),
         pytest.param("http://[::1]/", id="loopback-v6"),
         pytest.param("http://10.0.0.1/", id="rfc1918-10"),
         pytest.param("http://192.168.1.1/", id="rfc1918-192"),
@@ -145,3 +149,17 @@ def test_validate_mcp_url_combines_both_checks(monkeypatch: pytest.MonkeyPatch)
         asyncio.run(validate_mcp_url("http://169.254.169.254/"))
 
     assert asyncio.run(validate_mcp_url("https://example.com/")) == "https://example.com/"
+
+
+def test_private_network_resolves_decoded_public_hostname(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv(PRIVATE_NET_ENV, raising=False)
+    hosts: list[str] = []
+
+    def resolve(hostname: str) -> bool:
+        hosts.append(hostname)
+        return False
+
+    monkeypatch.setattr("webskrap.urls._resolve_blocked", resolve)
+    url = "https://%65xample.com/path%20name"
+    assert asyncio.run(validate_mcp_url(url)) == url
+    assert hosts == ["example.com"]

{webskrap-3.2.2 → webskrap-3.2.3}/uv.lock

@@ -1163,7 +1163,7 @@ wheels = [
 
 [[package]]
 name = "webskrap"
-version = "3.2.2"
+version = "3.2.3"
 source = { editable = "." }
 dependencies = [
     { name = "mcp" },