webget-cli 0.14.0__tar.gz → 0.15.0__tar.gz

{webget_cli-0.14.0/webget_cli.egg-info → webget_cli-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.14.0
+Version: 0.15.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -103,7 +103,7 @@ webget --help
 
 ### Browser runtime (optional)
 
-Crawl4AI drives a Playwright Chromium. `pip install "webget-cli[browser]"`
+Crawl4AI drives a Playwright browser. `pip install "webget-cli[browser]"`
 installs the Python packages; the browser binary itself is downloaded
 separately:
 
@@ -111,6 +111,41 @@ separately:
 python -m playwright install chromium
 ```
 
+**You may not need that download.** Before reaching for Playwright's bundled
+Chromium, webget looks for a Chromium-family browser already installed on the
+machine and uses it when it can actually be driven. Run `webget doctor` to see
+what was found and what will be used:
+
+```bash
+webget doctor
+```
+
+Resolution order (first match wins):
+
+| Step | Source | Notes |
+|---|---|---|
+| 1 | `WEBGET_BROWSER_CDP` | Attach to a running browser over CDP. The only route that works for browsers with no Playwright channel. |
+| 2 | `WEBGET_BROWSER_CHANNEL` | A Playwright channel: `chrome`, `msedge`, or `chromium`. |
+| 3 | `WEBGET_BROWSER_PATH` | An explicit binary. Accepted, but see the limitation below. |
+| 4 | Auto-detected browser | Chrome, Edge, Chromium, Brave, Vivaldi or Opera found on the machine. |
+| 5 | Playwright bundled Chromium | The fallback when nothing usable is found. |
+
+Two limitations worth knowing, both reported honestly by `webget doctor`
+instead of failing quietly:
+
+- **Brave, Vivaldi and Opera cannot be launched by crawl4ai.** Playwright has no
+  channel for them, and crawl4ai 0.9.2 does not forward `executable_path` to
+  Playwright, so there is no path to the binary. They are detected and reported
+  as NOT USABLE. The workaround is CDP: start the browser with
+  `--remote-debugging-port=9222`, then set
+  `WEBGET_BROWSER_CDP=http://127.0.0.1:9222`.
+- **CDP is opt-in on purpose.** Attaching to a browser you are already logged
+  into mixes your personal session cookies into crawl output, so webget never
+  auto-detects an open debugging port.
+
+System Firefox is never selected: Playwright drives Firefox over WebDriver BiDi
+with its own patched build.
+
 Without the browser extra, `webget` still works for search and plain HTTP
 fetches. A fetch that needs the browser (JS rendering, `--profile` sessions,
 `login`) prints a clear warning telling you how to install it.

{webget_cli-0.14.0 → webget_cli-0.15.0}/README.md

@@ -66,7 +66,7 @@ webget --help
 
 ### Browser runtime (optional)
 
-Crawl4AI drives a Playwright Chromium. `pip install "webget-cli[browser]"`
+Crawl4AI drives a Playwright browser. `pip install "webget-cli[browser]"`
 installs the Python packages; the browser binary itself is downloaded
 separately:
 
@@ -74,6 +74,41 @@ separately:
 python -m playwright install chromium
 ```
 
+**You may not need that download.** Before reaching for Playwright's bundled
+Chromium, webget looks for a Chromium-family browser already installed on the
+machine and uses it when it can actually be driven. Run `webget doctor` to see
+what was found and what will be used:
+
+```bash
+webget doctor
+```
+
+Resolution order (first match wins):
+
+| Step | Source | Notes |
+|---|---|---|
+| 1 | `WEBGET_BROWSER_CDP` | Attach to a running browser over CDP. The only route that works for browsers with no Playwright channel. |
+| 2 | `WEBGET_BROWSER_CHANNEL` | A Playwright channel: `chrome`, `msedge`, or `chromium`. |
+| 3 | `WEBGET_BROWSER_PATH` | An explicit binary. Accepted, but see the limitation below. |
+| 4 | Auto-detected browser | Chrome, Edge, Chromium, Brave, Vivaldi or Opera found on the machine. |
+| 5 | Playwright bundled Chromium | The fallback when nothing usable is found. |
+
+Two limitations worth knowing, both reported honestly by `webget doctor`
+instead of failing quietly:
+
+- **Brave, Vivaldi and Opera cannot be launched by crawl4ai.** Playwright has no
+  channel for them, and crawl4ai 0.9.2 does not forward `executable_path` to
+  Playwright, so there is no path to the binary. They are detected and reported
+  as NOT USABLE. The workaround is CDP: start the browser with
+  `--remote-debugging-port=9222`, then set
+  `WEBGET_BROWSER_CDP=http://127.0.0.1:9222`.
+- **CDP is opt-in on purpose.** Attaching to a browser you are already logged
+  into mixes your personal session cookies into crawl output, so webget never
+  auto-detects an open debugging port.
+
+System Firefox is never selected: Playwright drives Firefox over WebDriver BiDi
+with its own patched build.
+
 Without the browser extra, `webget` still works for search and plain HTTP
 fetches. A fetch that needs the browser (JS rendering, `--profile` sessions,
 `login`) prints a clear warning telling you how to install it.

{webget_cli-0.14.0 → webget_cli-0.15.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.14.0"
+version = "0.15.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

{webget_cli-0.14.0 → webget_cli-0.15.0}/tests/test_adversarial_http.py

@@ -166,9 +166,14 @@ class TestResponseBodies:
         assert _one(res)["status"] == "success"  # httpx auto-decompresses
 
     def test_huge_response_is_bounded(self, fresh_cache):
+        from webget.truncate import ELLIPSIS_MARKER
+
         server = fresh_cache
         # 5MB body; scrape_many must truncate, not blow memory or hang.
         res = asyncio.run(_fetch(server.url("/huge"), max_chars=1000))
         out = _one(res)
         assert out["status"] == "success"
-        assert len(out["markdown"]) <= 1000
+        # smart_truncate cuts at the limit and appends the ellipsis marker
+        # (this body has no clean boundary, so it is a hard cut at the limit).
+        assert len(out["markdown"]) <= 1000 + len(ELLIPSIS_MARKER)
+        assert out["markdown"].endswith(ELLIPSIS_MARKER)

webget_cli-0.15.0/tests/test_browser_discovery.py

@@ -0,0 +1,270 @@
+"""Tests for browser discovery and the ladder's use of it.
+
+No Chrome needed: everything here stubs the filesystem probe or checks the
+precedence rules directly. The point is that the resolution ORDER and the
+usability verdict are the product, not the individual detections.
+
+The usability rules encode a hard constraint discovered by running the real
+crawl4ai 0.9.2:
+
+    browser_manager._build_browser_args() returns
+        {"headless": ..., "args": ..., "channel": ...}
+    and then calls playwright.chromium.launch(**browser_args).
+
+There is no executable_path in that dict, so a browser without a Playwright
+channel (Brave, Vivaldi, Opera) cannot be launched at all, even though
+Playwright itself supports executable_path. BrowserConfig(...) even raises
+TypeError on executable_path. These tests pin that reality down so a future
+crawl4ai upgrade that fixes it will fail loudly here instead of silently.
+"""
+from __future__ import annotations
+
+import os
+import unittest
+from unittest import mock
+
+from webget import browser as browser_mod
+
+
+class ScanTest(unittest.TestCase):
+    def test_brave_has_no_playwright_channel(self):
+        with mock.patch("shutil.which") as which, mock.patch.object(
+            browser_mod, "_macos_bundle", return_value=None
+        ):
+            which.side_effect = lambda b: "/usr/bin/brave" if b == "brave-browser" else None
+            found = browser_mod.scan_installed_browsers()
+        self.assertEqual(found, [("brave", None, "/usr/bin/brave")])
+
+    def test_chrome_maps_to_channel(self):
+        with mock.patch("shutil.which") as which, mock.patch.object(
+            browser_mod, "_macos_bundle", return_value=None
+        ):
+            which.side_effect = lambda b: "/usr/bin/google-chrome" if b == "google-chrome" else None
+            found = browser_mod.scan_installed_browsers()
+        self.assertEqual(found, [("chrome", "chrome", "/usr/bin/google-chrome")])
+
+    def test_chrome_preferred_over_chromium(self):
+        with mock.patch("shutil.which") as which, mock.patch.object(
+            browser_mod, "_macos_bundle", return_value=None
+        ):
+            which.side_effect = lambda b: {
+                "google-chrome": "/usr/bin/google-chrome",
+                "chromium": "/usr/bin/chromium",
+            }.get(b)
+            found = browser_mod.scan_installed_browsers()
+        self.assertEqual([f[0] for f in found], ["chrome", "chromium"])
+
+    def test_firefox_is_never_selected(self):
+        """System Firefox cannot be driven by Playwright; it must not appear."""
+        with mock.patch("shutil.which") as which, mock.patch.object(
+            browser_mod, "_macos_bundle", return_value=None
+        ):
+            which.side_effect = lambda b: "/usr/bin/firefox" if b == "firefox" else None
+            self.assertEqual(browser_mod.scan_installed_browsers(), [])
+
+    def test_nothing_found_is_empty(self):
+        with mock.patch("shutil.which", return_value=None), mock.patch.object(
+            browser_mod, "_macos_bundle", return_value=None
+        ):
+            self.assertEqual(browser_mod.scan_installed_browsers(), [])
+
+
+class UsabilityTest(unittest.TestCase):
+    """Detected != usable. These pin the crawl4ai constraint."""
+
+    def test_brave_detected_but_reported_unusable(self):
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("brave", None, "/usr/bin/brave")],
+        ):
+            choice = browser_mod.resolve_browser()
+        self.assertFalse(choice.usable)
+        self.assertEqual(choice.to_browser_config_kwargs(), {})
+        self.assertIn("brave", choice.describe().lower())
+        self.assertIn("NOT usable", choice.describe())
+
+    def test_unusable_caveat_names_the_way_out(self):
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("brave", None, "/usr/bin/brave")],
+        ):
+            choice = browser_mod.resolve_browser()
+        self.assertIn("WEBGET_BROWSER_CDP", choice.caveat)
+        self.assertIn("remote-debugging-port", choice.caveat)
+
+    def test_falls_back_to_bundled_when_nothing_found(self):
+        with mock.patch.object(browser_mod, "scan_installed_browsers", return_value=[]):
+            choice = browser_mod.resolve_browser()
+        self.assertEqual(choice.source, "bundled")
+        self.assertTrue(choice.usable)
+        self.assertEqual(choice.to_browser_config_kwargs(), {})
+        self.assertEqual(choice.describe(), "playwright bundled chromium")
+
+    def test_usable_browser_wins_over_unusable_one(self):
+        """If both Brave and Chrome exist, pick Chrome, do not stop at Brave."""
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("brave", None, "/usr/bin/brave"), ("chrome", "chrome", "/usr/bin/google-chrome")],
+        ):
+            choice = browser_mod.resolve_browser()
+        self.assertTrue(choice.usable)
+        self.assertEqual(choice.channel, "chrome")
+
+
+class ConfigKwargsTest(unittest.TestCase):
+    """crawl4ai forwards chrome_channel and cdp_url, nothing else."""
+
+    def test_channel_emits_chrome_channel(self):
+        choice = browser_mod.BrowserChoice(name="chrome", channel="chrome")
+        self.assertEqual(choice.to_browser_config_kwargs(), {"chrome_channel": "chrome"})
+
+    def test_default_chromium_emits_nothing(self):
+        """crawl4ai skips the default value because it breaks Windows."""
+        choice = browser_mod.BrowserChoice(name="chromium", channel="chromium")
+        self.assertEqual(choice.to_browser_config_kwargs(), {})
+
+    def test_path_never_emits_executable_path(self):
+        """BrowserConfig raises TypeError on executable_path in 0.9.2."""
+        choice = browser_mod.BrowserChoice(name="brave", path="/usr/bin/brave")
+        self.assertNotIn("executable_path", choice.to_browser_config_kwargs())
+        self.assertEqual(choice.to_browser_config_kwargs(), {})
+
+    def test_cdp_emits_cdp_url(self):
+        choice = browser_mod.BrowserChoice(name="cdp", cdp_url="http://127.0.0.1:9222")
+        self.assertEqual(choice.to_browser_config_kwargs(), {"cdp_url": "http://127.0.0.1:9222"})
+
+    def test_cdp_wins_over_channel(self):
+        choice = browser_mod.BrowserChoice(name="cdp", channel="chrome", cdp_url="http://x:1")
+        self.assertEqual(choice.to_browser_config_kwargs(), {"cdp_url": "http://x:1"})
+
+    def test_empty_when_bundled(self):
+        self.assertEqual(browser_mod.BrowserChoice(name="chromium").to_browser_config_kwargs(), {})
+
+
+class ResolvePrecedenceTest(unittest.TestCase):
+    def setUp(self):
+        self._saved = dict(os.environ)
+        for key in ("WEBGET_BROWSER_PATH", "WEBGET_BROWSER_CHANNEL", "WEBGET_BROWSER_CDP"):
+            os.environ.pop(key, None)
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._saved)
+
+    def test_cdp_wins_over_everything(self):
+        os.environ["WEBGET_BROWSER_CDP"] = "http://127.0.0.1:9222"
+        os.environ["WEBGET_BROWSER_CHANNEL"] = "chrome"
+        choice = browser_mod.resolve_browser()
+        self.assertEqual(choice.cdp_url, "http://127.0.0.1:9222")
+        self.assertEqual(choice.source, "env:WEBGET_BROWSER_CDP")
+
+    def test_env_channel_wins_over_detection(self):
+        os.environ["WEBGET_BROWSER_CHANNEL"] = "msedge"
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("brave", None, "/usr/bin/brave")],
+        ):
+            choice = browser_mod.resolve_browser()
+        self.assertEqual(choice.channel, "msedge")
+        self.assertEqual(choice.source, "env:WEBGET_BROWSER_CHANNEL")
+
+    def test_unsupported_env_channel_is_reported_not_ignored(self):
+        """Asking for a channel Playwright does not have must not be silent."""
+        os.environ["WEBGET_BROWSER_CHANNEL"] = "brave"
+        choice = browser_mod.resolve_browser()
+        self.assertFalse(choice.usable)
+        self.assertIn("brave", choice.caveat.lower())
+        self.assertIn("WEBGET_BROWSER_CDP", choice.caveat)
+
+    def test_missing_env_path_raises_instead_of_falling_through(self):
+        """A typo must not silently become 'use the bundled browser'."""
+        os.environ["WEBGET_BROWSER_PATH"] = "/nope/does-not-exist"
+        with mock.patch("os.path.exists", return_value=False), self.assertRaises(
+            ValueError
+        ) as ctx:
+            browser_mod.resolve_browser()
+        self.assertIn("/nope/does-not-exist", str(ctx.exception))
+
+    def test_env_path_is_honest_about_being_unsupported(self):
+        """We accept the var but must say it cannot be honoured by crawl4ai."""
+        os.environ["WEBGET_BROWSER_PATH"] = "/opt/my/brave"
+        with mock.patch("os.path.exists", return_value=True):
+            choice = browser_mod.resolve_browser()
+        self.assertFalse(choice.usable)
+        self.assertIn("executable_path", choice.caveat)
+
+    def test_detected_channel_browser_beats_bundled(self):
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("chrome", "chrome", "/usr/bin/google-chrome")],
+        ):
+            choice = browser_mod.resolve_browser()
+        self.assertTrue(choice.is_external)
+        self.assertTrue(choice.usable)
+
+
+class CdpTest(unittest.TestCase):
+    def test_cdp_is_opt_in_only(self):
+        """Attaching to a logged-in browser leaks the user's session; never auto."""
+        saved = os.environ.pop("WEBGET_BROWSER_CDP", None)
+        try:
+            self.assertIsNone(browser_mod.cdp_endpoint())
+            os.environ["WEBGET_BROWSER_CDP"] = "http://127.0.0.1:9222"
+            self.assertEqual(browser_mod.cdp_endpoint(), "http://127.0.0.1:9222")
+        finally:
+            os.environ.pop("WEBGET_BROWSER_CDP", None)
+            if saved is not None:
+                os.environ["WEBGET_BROWSER_CDP"] = saved
+
+    def test_resolve_does_not_auto_detect_cdp(self):
+        saved = os.environ.pop("WEBGET_BROWSER_CDP", None)
+        try:
+            with mock.patch.object(browser_mod, "scan_installed_browsers", return_value=[]):
+                choice = browser_mod.resolve_browser()
+            self.assertEqual(choice.source, "bundled")
+            self.assertIsNone(choice.cdp_url)
+        finally:
+            if saved is not None:
+                os.environ["WEBGET_BROWSER_CDP"] = saved
+
+
+class LadderWiringTest(unittest.TestCase):
+    """The ladder must pass the resolved kwargs into BrowserConfig."""
+
+    def test_ladder_passes_resolved_kwargs(self):
+        import inspect
+
+        from webget import ladder
+
+        src = inspect.getsource(ladder)
+        self.assertIn("_browser_module.resolve_browser()", src)
+        self.assertIn("**bc_kwargs", src)
+
+    def test_ladder_does_not_pass_executable_path(self):
+        """BrowserConfig rejects it, so it must never reach the constructor."""
+        import inspect
+
+        from webget import ladder
+
+        src = inspect.getsource(ladder)
+        self.assertNotIn("executable_path=", src)
+
+    def test_ladder_reports_a_bad_env_path_as_a_reason(self):
+        import inspect
+
+        from webget import ladder
+
+        src = inspect.getsource(ladder)
+        self.assertIn("except ValueError as exc:", src)
+
+    def test_ladder_warns_when_browser_is_unusable(self):
+        import inspect
+
+        from webget import ladder
+
+        src = inspect.getsource(ladder)
+        self.assertIn("choice.usable", src)
+
+
+if __name__ == "__main__":
+    unittest.main()

webget_cli-0.15.0/tests/test_doctor.py

@@ -0,0 +1,119 @@
+"""Tests for `webget doctor`.
+
+The command exists so a user can see WHY a crawl used (or ignored) a browser.
+These tests pin the shape of the report and, more importantly, that it never
+claims something is usable when the launcher cannot drive it.
+"""
+from __future__ import annotations
+
+import json
+import os
+import unittest
+from contextlib import redirect_stdout
+from io import StringIO
+from unittest import mock
+
+from webget import browser as browser_mod
+from webget import cli
+
+
+def _run_doctor(json_out=False):
+    buf = StringIO()
+    with redirect_stdout(buf):
+        cli.cmd_doctor(json_out=json_out)
+    return buf.getvalue()
+
+
+class DoctorOutputTest(unittest.TestCase):
+    def setUp(self):
+        self._saved = dict(os.environ)
+        for key in ("WEBGET_BROWSER_CDP", "WEBGET_BROWSER_CHANNEL", "WEBGET_BROWSER_PATH"):
+            os.environ.pop(key, None)
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._saved)
+
+    def test_text_output_names_the_selected_browser(self):
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("chrome", "chrome", "/usr/bin/google-chrome")],
+        ):
+            out = _run_doctor()
+        self.assertIn("webget doctor", out)
+        self.assertIn("chrome", out)
+        self.assertIn("usable", out)
+
+    def test_unusable_browser_is_flagged_and_explained(self):
+        with mock.patch.object(
+            browser_mod, "scan_installed_browsers",
+            return_value=[("brave", None, "/usr/bin/brave")],
+        ):
+            out = _run_doctor()
+        self.assertIn("NOT USABLE", out)
+        self.assertIn("WEBGET_BROWSER_CDP", out)
+        self.assertIn("no playwright channel", out)
+
+    def test_reports_bundled_fallback_when_nothing_found(self):
+        with mock.patch.object(browser_mod, "scan_installed_browsers", return_value=[]):
+            out = _run_doctor()
+        self.assertIn("playwright bundled chromium", out)
+        self.assertIn("detected on this machine: none", out)
+
+    def test_json_output_is_valid_and_complete(self):
+        with mock.patch.object(browser_mod, "scan_installed_browsers", return_value=[]):
+            out = _run_doctor(json_out=True)
+        report = json.loads(out)
+        for key in ("browser", "crawl4ai", "playwright_cache", "env"):
+            self.assertIn(key, report)
+        self.assertIn("selected", report["browser"])
+        self.assertIn("usable", report["browser"])
+
+    def test_env_overrides_are_shown(self):
+        os.environ["WEBGET_BROWSER_CDP"] = "http://127.0.0.1:9222"
+        out = _run_doctor()
+        self.assertIn("WEBGET_BROWSER_CDP=http://127.0.0.1:9222", out)
+
+    def test_bad_env_path_reports_error_without_crashing(self):
+        """A bad WEBGET_BROWSER_PATH must surface as an error line.
+
+        Note: do NOT mock os.path.exists globally here. crawl4ai's import chain
+        calls dotenv.find_dotenv(), which uses os.path.exists internally and
+        asserts on the result; a blanket mock breaks the import and the failure
+        looks like a doctor bug rather than a test artifact.
+        """
+        os.environ["WEBGET_BROWSER_PATH"] = "/nope/missing"
+        out = _run_doctor()
+        self.assertIn("ERROR", out)
+        self.assertIn("/nope/missing", out)
+
+    def test_doctor_does_not_require_crawl4ai(self):
+        """A missing optional dep must be reported, not raised."""
+        real_import = __builtins__["__import__"] if isinstance(__builtins__, dict) else __builtins__.__import__
+
+        def fake_import(name, *args, **kwargs):
+            if name == "crawl4ai":
+                raise ImportError("not installed")
+            return real_import(name, *args, **kwargs)
+
+        with mock.patch("builtins.__import__", side_effect=fake_import), mock.patch.object(
+            browser_mod, "scan_installed_browsers", return_value=[]
+        ):
+            out = _run_doctor()
+        self.assertIn("NOT installed", out)
+        self.assertIn("pip install webget-cli[browser]", out)
+
+
+class DoctorWiringTest(unittest.TestCase):
+    def test_doctor_is_dispatched_from_main(self):
+        import inspect
+
+        src = inspect.getsource(cli.main)
+        self.assertIn('cmd == "doctor"', src)
+
+    def test_doctor_appears_in_usage(self):
+        self.assertIn("webget doctor", cli.__doc__ or "")
+
+
+if __name__ == "__main__":
+    unittest.main()

webget_cli-0.15.0/tests/test_truncate.py

@@ -0,0 +1,88 @@
+import asyncio
+
+import webget_cli as webget
+from webget.truncate import smart_truncate
+
+MARKER = "\n\n[...truncated]"
+
+
+def test_short_text_passthrough():
+    assert smart_truncate("Short text.", 100) == "Short text."
+
+
+def test_exact_length_passthrough():
+    assert smart_truncate("abcde", 5) == "abcde"
+
+
+def test_paragraph_boundary_preferred():
+    text = "para one here\n\npara two here"
+    assert smart_truncate(text, 20) == "para one here" + MARKER
+
+
+def test_last_paragraph_boundary_wins():
+    text = "alpha\n\nbeta\n\ngamma"
+    assert smart_truncate(text, 15) == "alpha\n\nbeta" + MARKER
+
+
+def test_paragraph_boundary_beats_sentence():
+    # The paragraph break sits further along than the last sentence end in
+    # the first paragraph; the paragraph boundary must win.
+    text = "Alpha. Beta.\n\nGamma delta epsilon."
+    assert smart_truncate(text, 20) == "Alpha. Beta." + MARKER
+
+
+def test_sentence_fallback_when_no_paragraph():
+    text = "First sentence. Second sentence. Third sentence."
+    assert smart_truncate(text, 25) == "First sentence." + MARKER
+
+
+def test_sentence_fallback_exclamation_and_question():
+    text = "Wow! Great. Hmm? No way."
+    assert smart_truncate(text, 12) == "Wow! Great." + MARKER
+
+
+def test_sentence_fallback_newline_terminated():
+    assert smart_truncate("Ends here.\nAnd more text", 12) == "Ends here." + MARKER
+
+
+def test_word_fallback():
+    text = "one two three four five"
+    assert smart_truncate(text, 9) == "one two" + MARKER
+
+
+def test_hard_cut():
+    assert smart_truncate("abcdefghijkl", 5) == "abcde" + MARKER
+
+
+def test_marker_appended_exactly_once():
+    text = "alpha\n\nbeta\n\ngamma\n\ndelta"
+    result = smart_truncate(text, 20)
+    assert result.count(MARKER) == 1
+    assert result.endswith(MARKER)
+
+
+def test_ladder_path_marks_truncated_content(fresh_cache):
+    """Regression: the ladder's HTTP fast path must run scraped content
+    through smart_truncate, not a hard slice.
+
+    Exercises the real ladder path at unit level: scrape_many() ->
+    _resolve_fetch_http() -> fetch_http() (webget/http.py). The fetcher now
+    caps markdown with smart_truncate instead of markdown[:max_chars], so a
+    page that overflows the limit must come back already marked, and that
+    mark must survive to the final result (the CLI no longer re-slices it off).
+
+    max_chars is kept comfortably above the 100-char "thin content" floor so
+    the ladder reports success and returns the (truncated, marked) markdown.
+    """
+    server = fresh_cache
+    # /normal serves LONG_BODY (~1.4KB of repeating sentences) > max_chars.
+    url = server.url("/normal")
+    max_chars = 300
+    res = asyncio.run(
+        webget.scrape_many([url], max_chars=max_chars, strategy="http", no_cache=True)
+    )
+    out = res[url]
+    assert out["status"] == "success", out.get("error")
+    md = out["markdown"]
+    assert md.endswith(MARKER), "ladder output must carry the truncation marker"
+    assert md.count(MARKER) == 1