linkfetch 0.1.0__py3-none-any.whl

linkfetch/__init__.py

@@ -0,0 +1,3 @@
+"""linkfetch — local LinkedIn profile scraper emitting tailor-compatible YAML."""
+
+__version__ = "0.1.0"

linkfetch/capture/__init__.py

@@ -0,0 +1 @@
+"""Phase 1: capture rendered HTML from LinkedIn via a real browser session."""

linkfetch/capture/browser.py

@@ -0,0 +1,112 @@
+"""Playwright session management for capturing LinkedIn HTML.
+
+Uses a **persistent browser context** rooted at ``user_data_dir`` so the login
+cookie survives between runs — you log in once via ``linkfetch login`` and later
+``capture`` runs reuse the session.
+
+Playwright is an optional dependency. Importing this module is fine without it;
+the ImportError only surfaces when you actually open a browser, with the same
+install hint tailor uses.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from pathlib import Path
+
+from linkfetch.capture.expand import expand_page
+from linkfetch.config import Config
+
+_PLAYWRIGHT_HINT = (
+    "Playwright is required to capture from LinkedIn but is not installed.\n"
+    "Install it with:\n"
+    "  uv pip install playwright && playwright install chromium"
+)
+
+
+class CaptureError(RuntimeError):
+    pass
+
+
+def _require_playwright():
+    try:
+        from playwright.sync_api import sync_playwright
+
+        return sync_playwright
+    except ImportError as exc:  # pragma: no cover - environment dependent
+        raise CaptureError(_PLAYWRIGHT_HINT) from exc
+
+
+@contextmanager
+def browser_session(cfg: Config):
+    """Yield a logged-in-capable :class:`Session` backed by a persistent context."""
+    sync_playwright = _require_playwright()
+    cfg.user_data_dir.mkdir(parents=True, exist_ok=True)
+    with sync_playwright() as p:
+        context = p.chromium.launch_persistent_context(
+            user_data_dir=str(cfg.user_data_dir),
+            headless=cfg.browser.headless,
+        )
+        try:
+            page = context.pages[0] if context.pages else context.new_page()
+            yield Session(cfg, context, page)
+        finally:
+            context.close()
+
+
+class Session:
+    def __init__(self, cfg: Config, context, page) -> None:
+        self.cfg = cfg
+        self.context = context
+        self.page = page
+
+    # -- auth --------------------------------------------------------------
+
+    def is_logged_in(self) -> bool:
+        """Heuristic: the feed page redirects to /login when not authenticated."""
+        self.page.goto(
+            f"{self.cfg.base_url}/feed/",
+            wait_until="domcontentloaded",
+            timeout=self.cfg.browser.nav_timeout * 1000,
+        )
+        url = self.page.url.lower()
+        return "/login" not in url and "/authwall" not in url and "/checkpoint" not in url
+
+    def wait_for_login(self, *, poll_seconds: int = 3, max_wait_seconds: int = 900) -> bool:
+        """Block until the user completes login in the visible window, or time out."""
+        self.page.goto(
+            f"{self.cfg.base_url}/login",
+            wait_until="domcontentloaded",
+            timeout=self.cfg.browser.nav_timeout * 1000,
+        )
+        waited = 0
+        while waited < max_wait_seconds:
+            self.page.wait_for_timeout(poll_seconds * 1000)
+            waited += poll_seconds
+            url = self.page.url.lower()
+            if all(tok not in url for tok in ("/login", "/authwall", "/checkpoint")):
+                return True
+        return False
+
+    # -- capture -----------------------------------------------------------
+
+    def capture(self, url: str) -> str:
+        """Navigate to ``url``, fully expand the page, and return rendered HTML."""
+        self.page.goto(
+            url,
+            wait_until="domcontentloaded",
+            timeout=self.cfg.browser.nav_timeout * 1000,
+        )
+        expand_page(
+            self.page,
+            scroll_pause_ms=self.cfg.browser.scroll_pause_ms,
+            max_scrolls=self.cfg.browser.max_scrolls,
+        )
+        return self.page.content()
+
+
+def save_capture(captures_dir: Path, name: str, html: str) -> Path:
+    captures_dir.mkdir(parents=True, exist_ok=True)
+    path = captures_dir / f"{name}.html"
+    path.write_text(html, encoding="utf-8")
+    return path

linkfetch/capture/expand.py

@@ -0,0 +1,61 @@
+"""Deterministic page-expansion routine, shared by all sections.
+
+A LinkedIn details page lazy-loads list items as you scroll and truncates long
+text behind "…see more" / "Show all" controls. To capture *everything* we:
+
+  1. Scroll to the bottom in steps until the page height stops growing
+     (bounded by ``max_scrolls``).
+  2. Repeatedly click every expansion control that's still visible, until none
+     remain (bounded to avoid loops).
+
+This is browser-bound and side-effecting; it deliberately contains no
+section-specific logic. Takes a Playwright ``Page``.
+"""
+
+from __future__ import annotations
+
+# Text on the controls that reveal hidden content. Matched case-insensitively
+# against button labels / inner text.
+_EXPAND_LABELS = ("see more", "show more", "…more", "see all", "show all")
+
+
+def expand_page(page, *, scroll_pause_ms: int, max_scrolls: int) -> None:
+    _scroll_to_bottom(page, scroll_pause_ms=scroll_pause_ms, max_scrolls=max_scrolls)
+    _click_all_expanders(page, scroll_pause_ms=scroll_pause_ms)
+    # A second scroll pass: expanding text can reveal more lazy content.
+    _scroll_to_bottom(page, scroll_pause_ms=scroll_pause_ms, max_scrolls=max_scrolls)
+
+
+def _scroll_to_bottom(page, *, scroll_pause_ms: int, max_scrolls: int) -> None:
+    last_height = -1
+    for _ in range(max_scrolls):
+        height = page.evaluate("document.body.scrollHeight")
+        if height == last_height:
+            break
+        last_height = height
+        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        page.wait_for_timeout(scroll_pause_ms)
+    # Return to top so subsequent content is laid out.
+    page.evaluate("window.scrollTo(0, 0)")
+
+
+def _click_all_expanders(page, *, scroll_pause_ms: int, max_rounds: int = 6) -> None:
+    for _ in range(max_rounds):
+        clicked = 0
+        for button in page.query_selector_all("button"):
+            try:
+                if not button.is_visible():
+                    continue
+                label = (button.inner_text() or "").strip().lower()
+                aria = (button.get_attribute("aria-label") or "").lower()
+                blob = f"{label} {aria}"
+                if any(token in blob for token in _EXPAND_LABELS):
+                    button.click(timeout=1500)
+                    clicked += 1
+                    page.wait_for_timeout(150)
+            except Exception:
+                # Buttons can detach as the DOM updates; ignore and continue.
+                continue
+        if clicked == 0:
+            break
+        page.wait_for_timeout(scroll_pause_ms)

linkfetch/cli.py

@@ -0,0 +1,274 @@
+"""linkfetch CLI.
+
+Commands:
+  linkfetch doctor             — show config, Playwright availability, session state
+  linkfetch login              — open the browser and establish/refresh the session
+  linkfetch capture [opts]     — phase 1: save section HTML to data/captures
+  linkfetch parse  [opts]      — phase 2: HTML -> profile YAML (+ ZIP for THRIVE) in data/output
+  linkfetch run    [opts]      — capture then parse
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from linkfetch.config import Config, load_config
+from linkfetch.emit import dump_yaml
+from linkfetch.sections import Section, resolve
+
+# A Windows console on a legacy code page (cp1252, gbk, …) cannot encode "✓"
+# or "→"; printing one raised UnicodeEncodeError and killed the command.
+# Substitute what the console cannot show instead of crashing.
+for _stream in (sys.stdout, sys.stderr):
+    if hasattr(_stream, "reconfigure"):
+        _stream.reconfigure(errors="replace")
+
+app = typer.Typer(
+    help="Capture your own LinkedIn profile locally and turn it into structured profile files.",
+    no_args_is_help=True,
+)
+console = Console()
+err_console = Console(stderr=True)
+
+
+def _cfg() -> Config:
+    return load_config()
+
+
+def _fail(msg: str) -> None:
+    err_console.print(f"[bold red]Error:[/] {msg}")
+    raise typer.Exit(1)
+
+
+def _playwright_installed() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("playwright") is not None
+
+
+def _section_arg(sections: list[str] | None) -> list[Section]:
+    try:
+        return resolve(sections)
+    except ValueError as exc:
+        _fail(str(exc))
+        raise  # unreachable; for type-checkers
+
+
+# ----------------------------------------------------------------------------
+# doctor
+# ----------------------------------------------------------------------------
+
+
+@app.command()
+def doctor() -> None:
+    """Check configuration, Playwright availability, and login session."""
+    cfg = _cfg()
+    console.print(f"[bold]Data folder:[/]    {cfg.root}")
+    console.print(f"[bold]Captures dir:[/]   {cfg.captures_dir}")
+    console.print(f"[bold]Output dir:[/]     {cfg.output_dir}")
+    console.print(f"[bold]Browser profile:[/] {cfg.user_data_dir}")
+
+    if _playwright_installed():
+        console.print("[green]✓[/] Playwright is installed")
+    else:
+        console.print(
+            "[yellow]![/] Playwright not installed — capture/login unavailable.\n"
+            "    Reinstall linkfetch (Playwright is a dependency), then run: playwright install chromium"
+        )
+
+    has_session = cfg.user_data_dir.exists() and any(cfg.user_data_dir.iterdir())
+    if has_session:
+        console.print("[green]✓[/] A browser profile exists (you may already be logged in)")
+    else:
+        console.print("[yellow]![/] No browser profile yet — run `linkfetch login`")
+
+    console.print(f"[bold]Sections:[/] {', '.join(s.slug for s in resolve(None))}")
+
+
+# ----------------------------------------------------------------------------
+# login
+# ----------------------------------------------------------------------------
+
+
+@app.command()
+def login() -> None:
+    """Open a browser window so you can log in to LinkedIn once."""
+    cfg = _cfg()
+    from linkfetch.capture.browser import CaptureError, browser_session
+
+    if cfg.browser.headless:
+        _fail("Set browser.headless = false in config.toml to log in interactively.")
+
+    try:
+        with browser_session(cfg) as session:
+            if session.is_logged_in():
+                console.print("[green]✓[/] Already logged in. Session is ready.")
+                return
+            console.print(
+                "A browser window is open. Please log in to LinkedIn there.\n"
+                "Waiting up to 15 minutes for you to finish…"
+            )
+            if session.wait_for_login():
+                console.print("[green]✓[/] Login detected. Session saved.")
+            else:
+                _fail("Timed out waiting for login. Try `linkfetch login` again.")
+    except CaptureError as exc:
+        _fail(str(exc))
+
+
+# ----------------------------------------------------------------------------
+# capture (phase 1)
+# ----------------------------------------------------------------------------
+
+
+@app.command()
+def capture(
+    vanity: str = typer.Option(..., "--vanity", "-v", help="Your profile slug: linkedin.com/in/<vanity>"),
+    section: list[str] = typer.Option(None, "--section", "-s", help="Section slug(s); default all"),
+) -> None:
+    """Phase 1: log in (if needed) and save each section's rendered HTML."""
+    cfg = _cfg()
+    sections = _section_arg(section)
+    from linkfetch.capture.browser import CaptureError, browser_session, save_capture
+
+    try:
+        with browser_session(cfg) as session:
+            if not session.is_logged_in():
+                _fail("Not logged in. Run `linkfetch login` first.")
+
+            captured: dict[str, str] = {}
+            done_sources: set[str] = set()
+            for sec in sections:
+                if sec.capture_name in done_sources:
+                    continue  # awards + honors share one source page
+                url = _section_url(cfg, sec, vanity)
+                console.print(f"Capturing [bold]{sec.capture_name}[/] …")
+                html = session.capture(url)
+                path = save_capture(cfg.captures_dir, sec.capture_name, html)
+                captured[sec.capture_name] = str(path)
+                done_sources.add(sec.capture_name)
+
+            meta = {"vanity": vanity, "captured": captured}
+            (cfg.captures_dir / "meta.json").write_text(
+                json.dumps(meta, indent=2), encoding="utf-8"
+            )
+            console.print(
+                f"[green]✓[/] Saved {len(captured)} page(s) to {cfg.captures_dir}"
+            )
+    except CaptureError as exc:
+        _fail(str(exc))
+
+
+def _section_url(cfg: Config, sec: Section, vanity: str) -> str:
+    if sec.detail_path is None:  # basics: main profile page
+        return f"{cfg.base_url}/in/{vanity}/"
+    return f"{cfg.base_url}{sec.detail_path.format(vanity=vanity)}"
+
+
+# ----------------------------------------------------------------------------
+# parse (phase 2)
+# ----------------------------------------------------------------------------
+
+
+@app.command()
+def parse(
+    section: list[str] = typer.Option(None, "--section", "-s", help="Section slug(s); default all"),
+    zip_output: bool = typer.Option(
+        True, "--zip/--no-zip", help="Also bundle the profile files into one ZIP for THRIVE"
+    ),
+) -> None:
+    """Phase 2: parse captured HTML into YAML. No browser needed."""
+    cfg = _cfg()
+    sections = _section_arg(section)
+    if not cfg.captures_dir.exists():
+        _fail(f"No captures found at {cfg.captures_dir}. Run `linkfetch capture` first.")
+
+    table = Table(title="Parsed sections")
+    table.add_column("section")
+    table.add_column("items", justify="right")
+    table.add_column("output")
+    table.add_column("profile", justify="center")
+
+    any_written = False
+    profile_files: list[Path] = []
+    for sec in sections:
+        src = cfg.captures_dir / f"{sec.capture_name}.html"
+        if not src.is_file():
+            table.add_row(sec.slug, "—", "[dim]no capture[/]", "")
+            continue
+        html = src.read_text(encoding="utf-8")
+        try:
+            result = sec.parser(html)
+        except Exception as exc:  # one bad section must not kill the run
+            err_console.print(f"[yellow]war: {sec.slug} parser failed: {exc}[/]")
+            table.add_row(sec.slug, "[red]error[/]", "[dim]skipped[/]", "")
+            continue
+
+        out_path = cfg.output_dir / sec.out_file
+        count = 1 if not isinstance(result, list) else len(result)
+        dump_yaml(out_path, sec.top_key, result)
+        any_written = True
+        if sec.mapped:
+            profile_files.append(out_path)
+        table.add_row(
+            sec.slug,
+            str(count),
+            str(out_path.relative_to(cfg.root)) if _within(out_path, cfg.root) else str(out_path),
+            "✓" if sec.mapped else "",
+        )
+
+    console.print(table)
+    if any_written:
+        console.print(f"\n[green]✓[/] YAML written to {cfg.output_dir}")
+        if zip_output and profile_files:
+            bundle = write_profile_zip(cfg.output_dir / PROFILE_ZIP, profile_files)
+            console.print(
+                f"[green]✓[/] Profile bundle: {bundle}\n"
+                "Import it in THRIVE: Tailor → Profile → Import → choose this file."
+            )
+
+
+PROFILE_ZIP = "linkfetch-profile.zip"
+
+
+def write_profile_zip(target: Path, files: list[Path]) -> Path:
+    """Bundle profile YAML files flat into one ZIP, the form THRIVE imports."""
+    import zipfile
+
+    with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as bundle:
+        for path in files:
+            bundle.write(path, arcname=path.name)
+    return target
+
+
+def _within(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+# ----------------------------------------------------------------------------
+# run (capture + parse)
+# ----------------------------------------------------------------------------
+
+
+@app.command()
+def run(
+    vanity: str = typer.Option(..., "--vanity", "-v", help="Your profile slug: linkedin.com/in/<vanity>"),
+    section: list[str] = typer.Option(None, "--section", "-s", help="Section slug(s); default all"),
+) -> None:
+    """Convenience: capture then parse in one go."""
+    capture(vanity=vanity, section=section)
+    parse(section=section, zip_output=True)
+
+
+if __name__ == "__main__":
+    app()

linkfetch/config.py

@@ -0,0 +1,105 @@
+"""Configuration loading.
+
+Reads ``config.toml`` from the project root and exposes a typed ``Config``.
+The project root is discovered by walking upward from the current working
+directory looking for ``config.toml``, so a checkout works from any
+subdirectory.
+
+An installed copy (``uv tool install`` / ``pipx``) usually finds no
+``config.toml``. It then keeps everything — captured pages, output, and the
+logged-in browser profile — in the per-user data folder for the OS (see
+``user_data_root``), never inside the Python installation.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+try:  # Python 3.11+
+    import tomllib as _toml
+except ModuleNotFoundError:  # Python 3.10
+    import tomli as _toml  # type: ignore[no-redef]
+
+
+@dataclass(frozen=True)
+class BrowserConfig:
+    headless: bool
+    nav_timeout: int  # seconds
+    scroll_pause_ms: int
+    max_scrolls: int
+
+
+@dataclass(frozen=True)
+class Config:
+    root: Path
+    base_url: str
+    browser: BrowserConfig
+    captures_dir: Path
+    output_dir: Path
+    user_data_dir: Path
+
+
+def user_data_root() -> Path:
+    r"""The per-user data folder for linkfetch on this OS.
+
+    Windows: ``%LOCALAPPDATA%\linkfetch``; macOS:
+    ``~/Library/Application Support/linkfetch``; elsewhere:
+    ``$XDG_DATA_HOME/linkfetch`` or ``~/.local/share/linkfetch``.
+    ``LINKFETCH_HOME`` overrides all of them.
+    """
+    override = os.environ.get("LINKFETCH_HOME")
+    if override:
+        return Path(override).expanduser()
+    if sys.platform == "win32":
+        base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
+        return Path(base) / "linkfetch"
+    if sys.platform == "darwin":
+        return Path.home() / "Library" / "Application Support" / "linkfetch"
+    base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share")
+    return Path(base) / "linkfetch"
+
+
+def find_project_root(start: Path | None = None) -> Path:
+    """The dir containing ``config.toml`` above ``start``, else the user data folder."""
+    start = (start or Path.cwd()).resolve()
+    for candidate in [start, *start.parents]:
+        if (candidate / "config.toml").is_file():
+            return candidate
+    return user_data_root()
+
+
+def load_config(root: Path | None = None) -> Config:
+    root = find_project_root(root)
+    config_path = root / "config.toml"
+    data: dict = {}
+    if config_path.is_file():
+        with config_path.open("rb") as fh:
+            data = _toml.load(fh)
+
+    browser_raw = data.get("browser", {})
+    paths_raw = data.get("paths", {})
+    linkedin_raw = data.get("linkedin", {})
+
+    browser = BrowserConfig(
+        headless=bool(browser_raw.get("headless", False)),
+        nav_timeout=int(browser_raw.get("nav_timeout", 30)),
+        scroll_pause_ms=int(browser_raw.get("scroll_pause_ms", 600)),
+        max_scrolls=int(browser_raw.get("max_scrolls", 40)),
+    )
+
+    def resolve(key: str, default: str) -> Path:
+        raw = paths_raw.get(key, default)
+        p = Path(raw)
+        return p if p.is_absolute() else root / p
+
+    return Config(
+        root=root,
+        base_url=linkedin_raw.get("base_url", "https://www.linkedin.com"),
+        browser=browser,
+        captures_dir=resolve("captures", "data/captures"),
+        output_dir=resolve("output", "data/output"),
+        user_data_dir=resolve("user_data_dir", "data/browser"),
+    )

linkfetch/emit.py

@@ -0,0 +1,57 @@
+"""Serialize parsed models to YAML.
+
+Uses the same dumper settings as tailor's profile store so the mapped output
+files are stylistically identical to tailor's own YAML (and therefore drop in
+cleanly): ``sort_keys=False``, ``allow_unicode=True``, block style, width 100.
+
+Each output file wraps its list under a top-level key (``experiences:``,
+``skills:``, …), matching tailor's per-category file shape. ``basics`` is
+wrapped under ``basics:``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel
+
+
+def _to_plain(models: list[BaseModel] | BaseModel) -> object:
+    """Convert pydantic models to plain dicts, dropping None/empty values.
+
+    Dropping empties keeps the YAML clean and human-editable: a scraped item
+    with no projects/tags doesn't litter the file with ``tags: []``.
+    """
+    if isinstance(models, BaseModel):
+        return _prune(models.model_dump())
+    return [_prune(m.model_dump()) for m in models]
+
+
+def _prune(value: object) -> object:
+    if isinstance(value, dict):
+        pruned = {}
+        for k, v in value.items():
+            pv = _prune(v)
+            if pv is None or pv == [] or pv == {}:
+                continue
+            pruned[k] = pv
+        return pruned
+    if isinstance(value, list):
+        return [_prune(v) for v in value]
+    return value
+
+
+def dump_yaml(path: Path, top_key: str, models: list[BaseModel] | BaseModel) -> None:
+    """Write ``{top_key: <plain data>}`` to ``path`` as tailor-style YAML."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {top_key: _to_plain(models)}
+    with path.open("w", encoding="utf-8") as fh:
+        yaml.safe_dump(
+            payload,
+            fh,
+            sort_keys=False,
+            allow_unicode=True,
+            default_flow_style=False,
+            width=100,
+        )