linkfetch 0.1.0__tar.gz → 0.1.2__tar.gz

{linkfetch-0.1.0 → linkfetch-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: linkfetch
-Version: 0.1.0
+Version: 0.1.2
 Summary: Capture your own LinkedIn profile on your own computer and turn it into structured profile files
 Project-URL: Homepage, https://github.com/Prosperis/linkfetch
 Project-URL: Issues, https://github.com/Prosperis/linkfetch/issues
@@ -34,6 +34,11 @@ Description-Content-Type: text/markdown
 
 # linkfetch
 
+[![PyPI](https://img.shields.io/pypi/v/linkfetch)](https://pypi.org/project/linkfetch/)
+[![Python](https://img.shields.io/pypi/pyversions/linkfetch)](https://pypi.org/project/linkfetch/)
+[![Tests](https://github.com/Prosperis/linkfetch/actions/workflows/test.yml/badge.svg)](https://github.com/Prosperis/linkfetch/actions/workflows/test.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
+
 Capture **your own** LinkedIn profile on **your own computer** and turn it into
 structured profile files — every role, description, skill, project,
 certification and more — ready to import into
@@ -61,7 +66,7 @@ You need Python 3.10 or newer and [uv](https://docs.astral.sh/uv/) (or
 ```bash
 uv tool install linkfetch        # or: pipx install linkfetch
 linkfetch doctor                 # shows where data is kept and whether Playwright and a login exist
-playwright install chromium      # one-time download of the browser linkfetch drives
+linkfetch install-browser        # one-time download of the browser linkfetch drives (~150 MB)
 ```
 
 ## Use
@@ -108,6 +113,7 @@ Delete the folder to remove everything linkfetch stored.
 
 | Command | What it does |
 |---|---|
+| `linkfetch install-browser` | Download the browser linkfetch drives (one time, and again if an upgrade asks) |
 | `linkfetch doctor` | Show the data folders and check Playwright and your login |
 | `linkfetch login` | Open a browser window to log in to LinkedIn once |
 | `linkfetch capture --vanity <you>` | Save your profile pages (needs the browser) |
@@ -124,8 +130,10 @@ One file per section, each a list under a top-level key:
 `projects.yaml`, `certifications.yaml`, `awards.yaml`, `languages.yaml`,
 `courses.yaml`, `patents.yaml`, `recommendations.yaml`, `volunteering.yaml`,
 plus `honors.yaml` (a richer copy of awards, not included in the ZIP).
-Experiences keep their full description as a summary plus bullets, and roles
-at the same company stay grouped.
+Experiences keep their full description as bullets. Several roles at one
+company become separate experiences — each with its own title, dates and
+bullets — all under the company's name, because a role's dates and
+achievements are what tailoring selects from.
 
 ## How it works
 
@@ -143,7 +151,7 @@ at the same company stay grouped.
 git clone https://github.com/Prosperis/linkfetch
 cd linkfetch
 uv venv && uv pip install -e ".[dev]"
-playwright install chromium
+uv run linkfetch install-browser
 uv run pytest
 ```
 

{linkfetch-0.1.0 → linkfetch-0.1.2}/README.md

@@ -1,5 +1,10 @@
 # linkfetch
 
+[![PyPI](https://img.shields.io/pypi/v/linkfetch)](https://pypi.org/project/linkfetch/)
+[![Python](https://img.shields.io/pypi/pyversions/linkfetch)](https://pypi.org/project/linkfetch/)
+[![Tests](https://github.com/Prosperis/linkfetch/actions/workflows/test.yml/badge.svg)](https://github.com/Prosperis/linkfetch/actions/workflows/test.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
+
 Capture **your own** LinkedIn profile on **your own computer** and turn it into
 structured profile files — every role, description, skill, project,
 certification and more — ready to import into
@@ -27,7 +32,7 @@ You need Python 3.10 or newer and [uv](https://docs.astral.sh/uv/) (or
 ```bash
 uv tool install linkfetch        # or: pipx install linkfetch
 linkfetch doctor                 # shows where data is kept and whether Playwright and a login exist
-playwright install chromium      # one-time download of the browser linkfetch drives
+linkfetch install-browser        # one-time download of the browser linkfetch drives (~150 MB)
 ```
 
 ## Use
@@ -74,6 +79,7 @@ Delete the folder to remove everything linkfetch stored.
 
 | Command | What it does |
 |---|---|
+| `linkfetch install-browser` | Download the browser linkfetch drives (one time, and again if an upgrade asks) |
 | `linkfetch doctor` | Show the data folders and check Playwright and your login |
 | `linkfetch login` | Open a browser window to log in to LinkedIn once |
 | `linkfetch capture --vanity <you>` | Save your profile pages (needs the browser) |
@@ -90,8 +96,10 @@ One file per section, each a list under a top-level key:
 `projects.yaml`, `certifications.yaml`, `awards.yaml`, `languages.yaml`,
 `courses.yaml`, `patents.yaml`, `recommendations.yaml`, `volunteering.yaml`,
 plus `honors.yaml` (a richer copy of awards, not included in the ZIP).
-Experiences keep their full description as a summary plus bullets, and roles
-at the same company stay grouped.
+Experiences keep their full description as bullets. Several roles at one
+company become separate experiences — each with its own title, dates and
+bullets — all under the company's name, because a role's dates and
+achievements are what tailoring selects from.
 
 ## How it works
 
@@ -109,7 +117,7 @@ at the same company stay grouped.
 git clone https://github.com/Prosperis/linkfetch
 cd linkfetch
 uv venv && uv pip install -e ".[dev]"
-playwright install chromium
+uv run linkfetch install-browser
 uv run pytest
 ```
 

{linkfetch-0.1.0 → linkfetch-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "linkfetch"
-version = "0.1.0"
+version = "0.1.2"
 description = "Capture your own LinkedIn profile on your own computer and turn it into structured profile files"
 readme = "README.md"
 requires-python = ">=3.10"

{linkfetch-0.1.0 → linkfetch-0.1.2}/src/linkfetch/capture/browser.py

@@ -19,8 +19,12 @@ from linkfetch.config import Config
 
 _PLAYWRIGHT_HINT = (
     "Playwright is required to capture from LinkedIn but is not installed.\n"
-    "Install it with:\n"
-    "  uv pip install playwright && playwright install chromium"
+    "Reinstall linkfetch (Playwright is a dependency): uv tool install --force linkfetch"
+)
+
+_BROWSER_HINT = (
+    "The browser linkfetch drives is not downloaded yet (or Playwright was updated).\n"
+    "Run: linkfetch install-browser"
 )
 
 
@@ -43,10 +47,15 @@ def browser_session(cfg: Config):
     sync_playwright = _require_playwright()
     cfg.user_data_dir.mkdir(parents=True, exist_ok=True)
     with sync_playwright() as p:
-        context = p.chromium.launch_persistent_context(
-            user_data_dir=str(cfg.user_data_dir),
-            headless=cfg.browser.headless,
-        )
+        try:
+            context = p.chromium.launch_persistent_context(
+                user_data_dir=str(cfg.user_data_dir),
+                headless=cfg.browser.headless,
+            )
+        except Exception as exc:
+            if "executable doesn't exist" in str(exc).lower():
+                raise CaptureError(_BROWSER_HINT) from None
+            raise
         try:
             page = context.pages[0] if context.pages else context.new_page()
             yield Session(cfg, context, page)
@@ -90,19 +99,53 @@ class Session:
 
     # -- capture -----------------------------------------------------------
 
-    def capture(self, url: str) -> str:
-        """Navigate to ``url``, fully expand the page, and return rendered HTML."""
-        self.page.goto(
-            url,
-            wait_until="domcontentloaded",
-            timeout=self.cfg.browser.nav_timeout * 1000,
-        )
-        expand_page(
-            self.page,
-            scroll_pause_ms=self.cfg.browser.scroll_pause_ms,
-            max_scrolls=self.cfg.browser.max_scrolls,
+    def capture(self, url: str, *, attempts: int = 3) -> str:
+        """Navigate to ``url``, fully expand the page, and return rendered HTML.
+
+        LinkedIn sometimes navigates a details page while it is being read — a
+        late client-side redirect, or a control that turns out to be a link —
+        which destroys the page's JavaScript context mid-scroll. That is
+        retried: wait for the page to settle, load ``url`` again, start over.
+        """
+        last_error: Exception | None = None
+        for _ in range(attempts):
+            try:
+                self.page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=self.cfg.browser.nav_timeout * 1000,
+                )
+                try:
+                    self.page.wait_for_load_state(
+                        "load", timeout=self.cfg.browser.nav_timeout * 1000
+                    )
+                except Exception:
+                    pass  # a slow "load" event is not a reason to give up
+                expand_page(
+                    self.page,
+                    scroll_pause_ms=self.cfg.browser.scroll_pause_ms,
+                    max_scrolls=self.cfg.browser.max_scrolls,
+                )
+                return self.page.content()
+            except Exception as error:
+                if not is_navigation_race(error):
+                    raise
+                last_error = error
+                self.page.wait_for_timeout(2000)
+        raise CaptureError(
+            f"{url} kept navigating away while it was being read "
+            f"({attempts} attempts). Try again in a minute. Last error: {last_error}"
         )
-        return self.page.content()
+
+
+def is_navigation_race(error: Exception) -> bool:
+    """True for errors caused by the page navigating while a script ran."""
+    text = str(error).lower()
+    return (
+        "execution context was destroyed" in text
+        or "page is navigating" in text
+        or "frame was detached" in text
+    )
 
 
 def save_capture(captures_dir: Path, name: str, html: str) -> Path:

{linkfetch-0.1.0 → linkfetch-0.1.2}/src/linkfetch/capture/expand.py

@@ -26,17 +26,43 @@ def expand_page(page, *, scroll_pause_ms: int, max_scrolls: int) -> None:
     _scroll_to_bottom(page, scroll_pause_ms=scroll_pause_ms, max_scrolls=max_scrolls)
 
 
+# LinkedIn's current layout does not scroll the window: the window is exactly
+# one screen tall and the page scrolls inside a panel (`<main id="workspace">`).
+# Scrolling the window there loads nothing, and every list stopped at its
+# first batch (10 honors, 20 skills…). So scroll whichever element actually
+# scrolls — the one with the most hidden content — falling back to the window.
+_SCROLL_TARGET = """() => {
+  const scrollable = [...document.querySelectorAll('*')].filter((el) => {
+    const style = getComputedStyle(el);
+    return /(auto|scroll)/.test(style.overflowY) && el.scrollHeight > el.clientHeight + 50;
+  });
+  scrollable.sort((a, b) => (b.scrollHeight - b.clientHeight) - (a.scrollHeight - a.clientHeight));
+  return scrollable[0] || document.scrollingElement || document.documentElement;
+}"""
+
+_SCROLL_HEIGHT = f"() => ({_SCROLL_TARGET})().scrollHeight"
+_SCROLL_DOWN = f"() => {{ const el = ({_SCROLL_TARGET})(); el.scrollTop = el.scrollHeight; window.scrollTo(0, document.body.scrollHeight); }}"
+_SCROLL_UP = f"() => {{ const el = ({_SCROLL_TARGET})(); el.scrollTop = 0; window.scrollTo(0, 0); }}"
+
+
 def _scroll_to_bottom(page, *, scroll_pause_ms: int, max_scrolls: int) -> None:
     last_height = -1
+    unchanged = 0
     for _ in range(max_scrolls):
-        height = page.evaluate("document.body.scrollHeight")
+        height = page.evaluate(_SCROLL_HEIGHT)
         if height == last_height:
-            break
+            # LinkedIn fetches the next batch asynchronously; give it one more
+            # pause before deciding the list is complete.
+            unchanged += 1
+            if unchanged >= 2:
+                break
+        else:
+            unchanged = 0
         last_height = height
-        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        page.evaluate(_SCROLL_DOWN)
         page.wait_for_timeout(scroll_pause_ms)
     # Return to top so subsequent content is laid out.
-    page.evaluate("window.scrollTo(0, 0)")
+    page.evaluate(_SCROLL_UP)
 
 
 def _click_all_expanders(page, *, scroll_pause_ms: int, max_rounds: int = 6) -> None:

{linkfetch-0.1.0 → linkfetch-0.1.2}/src/linkfetch/cli.py

@@ -79,7 +79,7 @@ def doctor() -> None:
     else:
         console.print(
             "[yellow]![/] Playwright not installed — capture/login unavailable.\n"
-            "    Reinstall linkfetch (Playwright is a dependency), then run: playwright install chromium"
+            "    Reinstall linkfetch (Playwright is a dependency), then run: linkfetch install-browser"
         )
 
     has_session = cfg.user_data_dir.exists() and any(cfg.user_data_dir.iterdir())
@@ -96,6 +96,19 @@ def doctor() -> None:
 # ----------------------------------------------------------------------------
 
 
+@app.command("install-browser")
+def install_browser() -> None:
+    """Download the Chromium build linkfetch drives (one time, ~150 MB)."""
+    import subprocess
+
+    # Run Playwright through linkfetch's own Python: an installed tool
+    # (uv tool / pipx) exposes only the `linkfetch` command, not `playwright`.
+    result = subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"])
+    if result.returncode != 0:
+        _fail("The browser download failed. Check your connection and run it again.")
+    console.print("[green]✓[/] Browser ready. Next: linkfetch login")
+
+
 @app.command()
 def login() -> None:
     """Open a browser window so you can log in to LinkedIn once."""

linkfetch-0.1.2/tests/test_capture_retry.py

@@ -0,0 +1,68 @@
+"""Capture recovers when LinkedIn navigates a page while it is being read."""
+
+import pytest
+
+from linkfetch.capture.browser import CaptureError, Session, is_navigation_race
+from linkfetch.config import BrowserConfig, Config
+
+
+class FakePage:
+    def __init__(self, failures: int, error: str = "Page.evaluate: Execution context was destroyed"):
+        self.failures = failures
+        self.error = error
+        self.gotos = 0
+
+    def goto(self, url, **_):
+        self.gotos += 1
+
+    def wait_for_load_state(self, *_args, **_kwargs):
+        pass
+
+    def wait_for_timeout(self, _ms):
+        pass
+
+    def evaluate(self, script):
+        if self.failures:
+            self.failures -= 1
+            raise RuntimeError(self.error)
+        return 100
+
+    def query_selector_all(self, _selector):
+        return []
+
+    def content(self):
+        return "<html>ok</html>"
+
+
+def session(page):
+    cfg = Config(
+        root=None, base_url="https://www.linkedin.com",
+        browser=BrowserConfig(headless=True, nav_timeout=1, scroll_pause_ms=0, max_scrolls=2),
+        captures_dir=None, output_dir=None, user_data_dir=None,
+    )
+    return Session(cfg, context=None, page=page)
+
+
+def test_retries_after_a_navigation_race():
+    page = FakePage(failures=1)
+    assert session(page).capture("https://www.linkedin.com/in/x/details/education/") == "<html>ok</html>"
+    assert page.gotos == 2
+
+
+def test_gives_up_with_a_clear_error_after_repeated_races():
+    page = FakePage(failures=10)
+    with pytest.raises(CaptureError, match="kept navigating away"):
+        session(page).capture("https://www.linkedin.com/in/x/details/education/")
+    assert page.gotos == 3
+
+
+def test_other_errors_are_not_retried():
+    page = FakePage(failures=1, error="net::ERR_NAME_NOT_RESOLVED")
+    with pytest.raises(RuntimeError, match="ERR_NAME_NOT_RESOLVED"):
+        session(page).capture("https://www.linkedin.com/in/x/details/education/")
+    assert page.gotos == 1
+
+
+def test_recognizes_navigation_errors():
+    assert is_navigation_race(RuntimeError("Execution context was destroyed, most likely because of a navigation"))
+    assert not is_navigation_race(RuntimeError("Timeout 30000ms exceeded"))

{linkfetch-0.1.0 → linkfetch-0.1.2}/uv.lock

@@ -138,7 +138,7 @@ wheels = [
 
 [[package]]
 name = "linkfetch"
-version = "0.1.0"
+version = "0.1.2"
 source = { editable = "." }
 dependencies = [
     { name = "lxml" },