linkfetch 0.1.0__py3-none-any.whl

linkfetch/parse/common.py

@@ -0,0 +1,251 @@
+"""Shared helpers for section parsers.
+
+These target LinkedIn's **current** rendered DOM (captured via Playwright), which:
+
+* uses **obfuscated, per-build hashed class names** (``_759d3eea`` …) — so we never
+  key off classes;
+* puts visible text in ``<p>`` and ``<span>`` elements, frequently emitting the
+  **same string twice in a row** (a visible copy + a screen-reader copy);
+* lays out each section under ``<main>`` inside a single content ``<section>``
+  (the section title is its first visible fragment), with the footer, ad widgets
+  and global nav living *outside* that section;
+* renders each entry as a repeated sibling block (a ``<div>`` or ``<li>``) inside
+  that section — **not** the old single ``<ul>`` of ``<li>``s;
+* nests multiple roles at one company in an inner ``<ul>`` of role ``<li>``s
+  inside the company's block;
+* wraps long descriptions in a ``<span data-testid="expandable-text-box">`` whose
+  line breaks are real ``<br/>`` tags (which ``text_content()`` would otherwise
+  flatten).
+
+We anchor on this **structure** (``main`` → content ``section`` → repeating entry
+blocks → ordered text fragments), which is far more stable than the hashed
+classes.
+"""
+
+from __future__ import annotations
+
+import re
+
+from lxml import html as lhtml
+
+from linkfetch.text import clean
+
+# Chrome / nav / footer / ad boilerplate that leaks into the captured page.
+# Matched as a lowercase *substring* against an entry's head text so an entry
+# that is purely chrome is dropped.
+_CHROME = (
+    "home", "my network", "jobs", "messaging", "notifications",
+    "skip to search", "skip to main content", "skip to active conversation",
+    "back to search results", "why am i seeing this ad", "manage your ad",
+    "hide or report this ad", "report this ad", "ad options",
+    "linkedin corporation", "questions?", "visit our help center",
+    "manage your account and privacy", "go to your settings",
+    "recommendation transparency", "learn more about recommended content",
+    "select language", "profile language", "accessibility", "talent solutions",
+    "community guidelines", "marketing solutions", "privacy & terms",
+    "ad choices", "sales solutions", "small business", "safety center",
+)
+
+# Section-title fragments that appear as the first child block of the content
+# section and must not be treated as an entry.
+_SECTION_TITLES = (
+    "experience", "education", "skills", "projects",
+    "licenses & certifications", "licenses and certifications", "certifications",
+    "honors & awards", "honors and awards", "volunteering",
+    "recommendations", "patents", "courses", "languages",
+)
+
+
+def _root(page_html: str) -> lhtml.HtmlElement | None:
+    if not page_html or not page_html.strip():
+        return None
+    return lhtml.fromstring(page_html)
+
+
+def _content_section(page_html: str) -> lhtml.HtmlElement | None:
+    """Return the ``<main>``'s primary content ``<section>`` (the entry scope).
+
+    The captured page always carries one ``<section aria-label="Primary
+    content">`` — the real entries — alongside a fixed-size "People also
+    viewed" sidebar section that renders identically on every details page.
+    We take the labelled section outright when it is there. Falling back to
+    "most visible text fragments" picks the sidebar instead whenever a
+    person's real section (projects, certifications, honors, a short
+    education list, …) happens to be smaller than that sidebar's fixed
+    fragment count — not a rare case, since plenty of profiles are sparse in
+    exactly those sections. The fragment-count heuristic remains only as a
+    fallback for a page that, for whatever reason, has no labelled section.
+    """
+    root = _root(page_html)
+    if root is None:
+        return None
+    mains = root.xpath("//main")
+    scope = mains[0] if mains else root
+    sections = scope.xpath(".//section")
+    if not sections:
+        return scope
+    labelled = [s for s in sections if s.get("aria-label") == "Primary content"]
+    if labelled:
+        return labelled[0]
+    return max(sections, key=lambda s: len(ordered_texts(s)))
+
+
+def _is_chrome(text: str | None) -> bool:
+    if not text:
+        return True
+    low = text.lower()
+    return any(hint in low for hint in _CHROME)
+
+
+def entry_items(page_html: str) -> list[lhtml.HtmlElement]:
+    """Return the top-level entry blocks of a details page.
+
+    Finds the content ``<section>`` under ``<main>``, then the element whose
+    direct ``<div>``/``<li>``/``<section>`` children are the repeated entry
+    blocks. An entry child is one carrying at least two distinct visible text
+    fragments; we pick the container with the **most** such children (the
+    deepest one on ties — so we land on the real entry list rather than a
+    wrapper). Section-title and chrome blocks are then filtered out.
+    """
+    scope = _content_section(page_html)
+    if scope is None:
+        return []
+
+    best: lhtml.HtmlElement | None = None
+    best_n = 0
+    best_depth = -1
+    for el in scope.iter():
+        entries = [
+            c
+            for c in el
+            if c.tag in ("div", "li", "section") and len(ordered_texts(c)) >= 2
+        ]
+        n = len(entries)
+        if n == 0 or n < best_n:
+            continue
+        depth = sum(1 for _ in el.iterancestors())
+        if n > best_n or (n == best_n and depth > best_depth):
+            best_n = n
+            best = el
+            best_depth = depth
+
+    if best is None:
+        return []
+
+    items: list[lhtml.HtmlElement] = []
+    for kid in best:
+        if kid.tag not in ("div", "li", "section"):
+            continue
+        texts = ordered_texts(kid)
+        head = texts[0] if texts else None
+        if head is None or _is_chrome(head):
+            continue
+        if head.lower().strip() in _SECTION_TITLES:
+            continue
+        items.append(kid)
+    return items
+
+
+def _head_text(el: lhtml.HtmlElement) -> str | None:
+    for frag in ordered_texts(el):
+        return frag
+    return None
+
+
+def ordered_texts(el: lhtml.HtmlElement) -> list[str]:
+    """Ordered, de-duplicated visible text fragments inside ``el``.
+
+    Reads ``<p>`` and ``<span>`` text in document order, dropping consecutive
+    duplicates (LinkedIn's visible + screen-reader copies) and empties.
+    """
+    out: list[str] = []
+    for node in el.iter():
+        if node.tag not in ("p", "span"):
+            continue
+        txt = clean(node.text_content())
+        if not txt:
+            continue
+        if out and out[-1] == txt:
+            continue
+        out.append(txt)
+    return out
+
+
+def sub_entries(li: lhtml.HtmlElement) -> list[lhtml.HtmlElement]:
+    """Return nested sub-entry ``<li>``s (e.g. multiple roles at one company)."""
+    nested = li.xpath(".//ul/li")
+    return [n for n in nested if _head_text(n)]
+
+
+def first_link(el: lhtml.HtmlElement, *, contains: str | None = None) -> str | None:
+    """First anchor ``href`` inside ``el``; optionally requiring a substring."""
+    for a in el.xpath(".//a[@href]"):
+        href = a.get("href")
+        if href and (contains is None or contains in href):
+            return href.split("?")[0]
+    return None
+
+
+def description_text(el: lhtml.HtmlElement, *, exclude_linked: bool = False) -> str | None:
+    """Long-form description inside ``el``, preserving ``<br/>`` as newlines.
+
+    LinkedIn renders the description in a ``<span data-testid="expandable-text-box">``
+    using real ``<br/>`` tags for line breaks (which ``text_content()`` flattens).
+    We pick the longest such box and convert ``<br/>`` to newlines so the
+    description splits cleanly into bullets. Returns ``None`` when the entry has
+    no such box (i.e. genuinely has no description).
+
+    ``exclude_linked`` drops any box that sits inside an ``<a>``. On a details
+    page every box in scope is the one genuine description, so this is off by
+    default; the main profile page also renders every recent Activity post's
+    preview through this same component, each wrapped in a link to the post,
+    which "pick the longest box" would otherwise treat as a candidate for the
+    person's About text.
+    """
+    boxes = el.xpath('.//*[@data-testid="expandable-text-box"]')
+    if exclude_linked:
+        boxes = [b for b in boxes if not b.xpath("ancestor::a")]
+    best = ""
+    for box in boxes:
+        text = _text_with_breaks(box)
+        # Drop the trailing "… more" expand affordance LinkedIn appends.
+        text = re.sub(r"\s*…\s*more\s*$", "", text)
+        if len(text) > len(best):
+            best = text
+    return clean(best) or None
+
+
+def _text_with_breaks(el: lhtml.HtmlElement) -> str:
+    """``text_content()`` but with ``<br/>`` rendered as a newline."""
+    parts: list[str] = []
+    if el.text:
+        parts.append(el.text)
+    for child in el:
+        if child.tag == "br":
+            parts.append("\n")
+        else:
+            parts.append(_text_with_breaks(child))
+        if child.tail:
+            parts.append(child.tail)
+    return "".join(parts)
+
+
+def split_timeline(date_range: str | None) -> tuple[str | None, str | None]:
+    """Split ``"Mar 2021 - Present · 3 yrs"`` into ``(start, end)``.
+
+    The duration suffix after ``·`` is dropped. A single date with no separator
+    is treated as the start.
+    """
+    if not date_range:
+        return None, None
+    text = date_range.split("·")[0].strip()
+    for sep in (" – ", " - ", " — ", "–", "—", " to "):
+        if sep in text:
+            start, _, end = text.partition(sep)
+            return clean(start), clean(end)
+    return clean(text), None
+
+
+def looks_like_dates(text: str) -> bool:
+    """True if the fragment carries a 4-digit year or 'Present'."""
+    return bool(re.search(r"\b(19|20)\d{2}\b", text)) or "present" in text.lower()

linkfetch/parse/courses.py

@@ -0,0 +1,38 @@
+"""Parse the Courses details page.
+
+Ordered fragments per entry::
+
+    [name, "<course number>", "Associated with <school>"]
+
+e.g. ``["Algorithm Design and Analysis", "CSE 100", "Associated with …"]``.
+"""
+
+from __future__ import annotations
+
+from linkfetch.models import Course
+from linkfetch.parse.common import entry_items, ordered_texts
+from linkfetch.text import clean, slugify
+
+
+def parse(html: str) -> list[Course]:
+    out: list[Course] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        name = texts[0]
+        number = associated = None
+        for frag in texts[1:]:
+            if frag.lower().startswith("associated with"):
+                associated = clean(frag.split("with", 1)[-1])
+            elif number is None:
+                number = frag
+        out.append(
+            Course(
+                id=slugify(name, number),
+                name=name,
+                number=number,
+                associated_with=associated,
+            )
+        )
+    return out

linkfetch/parse/education.py

@@ -0,0 +1,51 @@
+"""Parse the Education details page.
+
+Ordered fragments per entry::
+
+    [school, "Degree, Field of study", "2016 – 2021", <activities/description>]
+
+The degree/field fragment is split on the first comma; the timeline fragment is
+a bare year range. The long-form "Activities and societies: …" / description
+text (an expandable box) becomes the bullets.
+"""
+
+from __future__ import annotations
+
+from linkfetch.models import Education, Timeline
+from linkfetch.parse.common import (
+    description_text,
+    entry_items,
+    looks_like_dates,
+    ordered_texts,
+    split_timeline,
+)
+from linkfetch.text import clean, slugify, split_bullets
+
+
+def parse(html: str) -> list[Education]:
+    out: list[Education] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        school = texts[0]
+
+        degree = field = None
+        start = end = None
+        for frag in texts[1:]:
+            if start is None and looks_like_dates(frag):
+                start, end = split_timeline(frag)
+            elif degree is None and not frag.lower().startswith("activities"):
+                degree, _, field = (clean(p) for p in frag.partition(","))
+
+        out.append(
+            Education(
+                id=slugify(school, degree),
+                school=school,
+                degree=degree,
+                field=field,
+                timeline=Timeline(start=start, end=end) if start else None,
+                bullets=split_bullets(description_text(el)),
+            )
+        )
+    return out

linkfetch/parse/experience.py

@@ -0,0 +1,129 @@
+"""Parse the Experience details page.
+
+Each entry block is a *company*. There are two shapes:
+
+* **Single role** — the block's ordered fragments are::
+
+    [title, "Org · Employment-type", "date range · duration", location,
+     <description>, "Skills: …"]
+
+* **Multiple roles** — the block heads with the company, then nests one role per
+  inner ``<li>``::
+
+    company head:  ["Roche", "Full-time · 5 yrs 4 mos", "Hybrid", …]
+    each role li:  [title, "date range · duration", location, <description>,
+                    "Skills: …"]
+
+We emit **one** :class:`Experience` per role; for multi-role companies the
+``org`` is the company head. The ``· duration`` / employment-type suffixes are
+stripped from ``org``; the description becomes ``bullets``; the trailing
+``Skills: …`` fragment is parsed into ``tech``.
+"""
+
+from __future__ import annotations
+
+from lxml import html as lhtml
+
+from linkfetch.models import Experience, Link, Position, Timeline
+from linkfetch.parse.common import (
+    description_text,
+    entry_items,
+    first_link,
+    looks_like_dates,
+    ordered_texts,
+    split_timeline,
+    sub_entries,
+)
+from linkfetch.text import clean, slugify, split_bullets
+
+
+def _strip_suffix(text: str | None) -> str | None:
+    """Drop the ``· …`` suffix LinkedIn appends (employment type, duration)."""
+    if not text:
+        return None
+    return clean(text.split("·")[0])
+
+
+def _is_skills(frag: str) -> bool:
+    return frag.lower().startswith("skills:")
+
+
+def _tech_from_skills(frag: str) -> list[str]:
+    """``"Skills: Angular, Python, +21 skills"`` -> ``["Angular", "Python"]``."""
+    body = frag.split(":", 1)[-1]
+    out: list[str] = []
+    for part in body.split(","):
+        item = clean(part)
+        if not item or item.lower().endswith("skills") or item.startswith("+"):
+            continue
+        out.append(item)
+    return out
+
+
+def parse(html: str) -> list[Experience]:
+    out: list[Experience] = []
+    for block in entry_items(html):
+        roles = sub_entries(block)
+        if roles:
+            org = _strip_suffix(ordered_texts(block)[0]) if ordered_texts(block) else None
+            href = first_link(block, contains="/company/")
+            for role in roles:
+                exp = _build(role, org=org, company_href=href)
+                if exp:
+                    out.append(exp)
+        else:
+            exp = _build(block, org=None, company_href=None)
+            if exp:
+                out.append(exp)
+    return out
+
+
+def _build(
+    el: lhtml.HtmlElement,
+    *,
+    org: str | None,
+    company_href: str | None,
+) -> Experience | None:
+    texts = ordered_texts(el)
+    if not texts:
+        return None
+
+    title = texts[0]
+    rest = texts[1:]
+
+    # For a single-role block the org is the 2nd fragment ("Org · Full-time").
+    if org is None and rest:
+        org = _strip_suffix(rest[0])
+        rest = rest[1:]
+    if not org:
+        return None
+
+    start = end = location = None
+    for frag in rest:
+        if _is_skills(frag):
+            continue
+        if start is None and looks_like_dates(frag):
+            start, end = split_timeline(frag)
+        elif location is None and not looks_like_dates(frag) and "\n" not in frag:
+            location = frag
+
+    bullets = split_bullets(description_text(el))
+    tech: list[str] = []
+    for frag in texts:
+        if _is_skills(frag):
+            tech = _tech_from_skills(frag)
+            break
+
+    href = company_href or first_link(el, contains="/company/")
+    links = [Link(label=org, url=href)] if href else []
+
+    return Experience(
+        id=slugify(org, title),
+        org=org,
+        location=location,
+        positions=[Position(title=title)],
+        timeline=Timeline(start=start, end=end) if start else None,
+        bullets=bullets,
+        tech=tech,
+        links=links,
+    )

linkfetch/parse/honors.py

@@ -0,0 +1,32 @@
+"""Parse Honors & awards into the richer extra Honor model.
+
+Same source section as awards.py (same fragment order), but kept as a standalone
+extra file so the full description survives in ``summary``. tailor's Award
+(awards.py) is the trimmed, mappable view.
+"""
+
+from __future__ import annotations
+
+from linkfetch.models import Honor
+from linkfetch.parse.awards import extract
+from linkfetch.parse.common import description_text, entry_items, ordered_texts
+from linkfetch.text import slugify
+
+
+def parse(html: str) -> list[Honor]:
+    out: list[Honor] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        title, issuer, date = extract(texts)
+        out.append(
+            Honor(
+                id=slugify(title, issuer),
+                title=title,
+                issuer=issuer,
+                date=date,
+                summary=description_text(el),
+            )
+        )
+    return out

linkfetch/parse/languages.py

@@ -0,0 +1,23 @@
+"""Parse the Languages details page.
+
+Ordered fragments per entry: ``[language name, proficiency]`` — e.g.
+``["English", "Full professional proficiency"]``.
+"""
+
+from __future__ import annotations
+
+from linkfetch.models import Language
+from linkfetch.parse.common import entry_items, ordered_texts
+from linkfetch.text import slugify
+
+
+def parse(html: str) -> list[Language]:
+    out: list[Language] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        name = texts[0]
+        proficiency = texts[1] if len(texts) > 1 else None
+        out.append(Language(id=slugify(name), name=name, proficiency=proficiency))
+    return out

linkfetch/parse/patents.py

@@ -0,0 +1,60 @@
+"""Parse the Patents details page.
+
+Ordered fragments per entry::
+
+    [title, "APPLICATION Filed · Filed Mar 26, 2025", <description>, …]
+
+The status ("Application Filed" / "Granted") and date are read from the second
+fragment; an explicit patent number (e.g. "US 1,234,567") is captured when
+present. The description is the expandable text box.
+"""
+
+from __future__ import annotations
+
+import re
+
+from linkfetch.models import Patent
+from linkfetch.parse.common import description_text, entry_items, ordered_texts
+from linkfetch.text import clean, slugify
+
+_DATE = re.compile(r"\b([A-Z][a-z]{2,8}\.?\s+\d{1,2},?\s+(?:19|20)\d{2}|(?:19|20)\d{2})\b")
+_NUMBER = re.compile(r"\b([A-Z]{2}\s?[\d,]{5,}|\d{1,3}[,/]\d{3}[,/\d]+)\b")
+
+
+def parse(html: str) -> list[Patent]:
+    out: list[Patent] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        title = texts[0]
+
+        number = status = date = None
+        meta = texts[1] if len(texts) > 1 else ""
+        if meta:
+            head = meta.split("·")[0]
+            if head and not _DATE.fullmatch(head.strip()):
+                cleaned = clean(head)
+                # "APPLICATION Filed" -> "Application Filed" (preserve real casing
+                # of mixed-case tokens; only fix all-caps words).
+                status = " ".join(
+                    w.capitalize() if w.isupper() else w for w in cleaned.split()
+                ) if cleaned else None
+            m = _DATE.search(meta)
+            if m:
+                date = clean(m.group(1).lstrip("Filed ").strip())
+            m = _NUMBER.search(meta)
+            if m:
+                number = clean(m.group(1))
+
+        out.append(
+            Patent(
+                id=slugify(title),
+                title=title,
+                number=number,
+                status=status,
+                date=date,
+                summary=description_text(el),
+            )
+        )
+    return out

linkfetch/parse/projects.py

@@ -0,0 +1,67 @@
+"""Parse the Projects details page.
+
+Ordered fragments per entry::
+
+    [name, "date range", <description>, <external link label>]
+
+The description is an expandable box whose first line is usually ``Role: …`` —
+we lift that into ``role`` and split the remainder into bullets. An associated
+external URL (e.g. a GitHub repo) is captured as a link.
+"""
+
+from __future__ import annotations
+
+from linkfetch.models import Link, Project, Timeline
+from linkfetch.parse.common import (
+    description_text,
+    entry_items,
+    looks_like_dates,
+    ordered_texts,
+    split_timeline,
+)
+from linkfetch.text import clean, slugify, split_bullets
+
+
+def parse(html: str) -> list[Project]:
+    out: list[Project] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        name = texts[0]
+
+        start = end = None
+        for frag in texts[1:]:
+            if looks_like_dates(frag):
+                start, end = split_timeline(frag)
+                break
+
+        role = None
+        bullets = split_bullets(description_text(el))
+        if bullets and bullets[0].lower().startswith("role:"):
+            role = clean(bullets[0].split(":", 1)[-1])
+            bullets = bullets[1:]
+
+        href = _external_link(el)
+        links = [Link(label=name, url=href)] if href else []
+
+        out.append(
+            Project(
+                id=slugify(name),
+                name=name,
+                role=role,
+                timeline=Timeline(start=start, end=end) if start else None,
+                bullets=bullets,
+                links=links,
+            )
+        )
+    return out
+
+
+def _external_link(el) -> str | None:
+    """First non-LinkedIn anchor (a real project/repo URL), if any."""
+    for a in el.xpath(".//a[@href]"):
+        href = a.get("href")
+        if href and href.startswith("http") and "linkedin.com" not in href:
+            return href.split("?")[0]
+    return None