linkfetch 0.1.0__py3-none-any.whl

linkfetch/models.py

@@ -0,0 +1,196 @@
+"""Models for scraped LinkedIn data.
+
+Two families:
+
+1. **tailor-mapped** models — a *subset* of tailor's profile models using the
+   **same field names** (``id``, ``summary``, ``bullets``, ``tags``, ``org``,
+   ``positions``, ``timeline``, …). YAML emitted from these drops straight into
+   tailor's profile store. We intentionally omit fields tailor fills in by hand
+   (e.g. multiple position framings, emphasis) — the scrape is deterministic and
+   leaves enrichment to the user.
+
+2. **extra** models — small purpose-built models for LinkedIn sections tailor
+   has no home for (education, volunteering, recommendations, …).
+
+All ``id`` values are stable slugs derived from the item's identity so re-runs
+produce diffable YAML.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+# ---------------------------------------------------------------------------
+# shared
+# ---------------------------------------------------------------------------
+
+
+class Timeline(BaseModel):
+    """A start/end span. ``end`` omitted or 'present' means ongoing.
+
+    Mirrors tailor.profile.models.Timeline (field-compatible).
+    """
+
+    start: str | None = None
+    end: str | None = None
+    note: str | None = None
+
+
+class Position(BaseModel):
+    """One title framing for a role. Mirrors tailor's Position.
+
+    The scrape produces exactly one framing (the real LinkedIn title); the user
+    adds alternative framings later in tailor.
+    """
+
+    title: str
+    seniority: str | None = None
+    emphasis: str | None = None
+    timeline: Timeline | None = None
+
+
+class Link(BaseModel):
+    label: str
+    url: str
+
+
+# ---------------------------------------------------------------------------
+# tailor-mapped models (field names match tailor.profile.models)
+# ---------------------------------------------------------------------------
+
+
+class Basics(BaseModel):
+    name: str
+    label: str | None = None  # headline
+    email: str | None = None
+    phone: str | None = None
+    location: str | None = None
+    summary: str | None = None  # the "About" section
+    links: list[Link] = Field(default_factory=list)
+
+
+class Experience(BaseModel):
+    id: str
+    org: str
+    location: str | None = None
+    positions: list[Position] = Field(default_factory=list)
+    timeline: Timeline | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+    tech: list[str] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+
+
+class Skill(BaseModel):
+    id: str
+    name: str
+    category: str | None = None
+    level: str | None = None
+    years: float | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+
+
+class Project(BaseModel):
+    id: str
+    name: str
+    role: str | None = None
+    timeline: Timeline | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+    tech: list[str] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+
+
+class Certification(BaseModel):
+    id: str
+    name: str
+    issuer: str | None = None
+    issued: str | None = None
+    expires: str | None = None
+    credential_id: str | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+
+
+class Award(BaseModel):
+    id: str
+    name: str
+    issuer: str | None = None
+    date: str | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# extra models (linkfetch-local schema; not in tailor)
+# ---------------------------------------------------------------------------
+
+
+class Education(BaseModel):
+    id: str
+    school: str
+    degree: str | None = None
+    field: str | None = None
+    timeline: Timeline | None = None
+    summary: str | None = None
+    bullets: list[str] = Field(default_factory=list)
+
+
+class Volunteering(BaseModel):
+    id: str
+    org: str
+    role: str | None = None
+    cause: str | None = None
+    timeline: Timeline | None = None
+    summary: str | None = None
+
+
+class Recommendation(BaseModel):
+    id: str
+    author: str
+    author_title: str | None = None
+    relationship: str | None = None
+    text: str | None = None
+
+
+class Patent(BaseModel):
+    id: str
+    title: str
+    number: str | None = None
+    status: str | None = None
+    date: str | None = None
+    summary: str | None = None
+
+
+class Course(BaseModel):
+    id: str
+    name: str
+    number: str | None = None
+    associated_with: str | None = None
+
+
+class Language(BaseModel):
+    id: str
+    name: str
+    proficiency: str | None = None
+
+
+class Honor(BaseModel):
+    """Honors & awards as a standalone extra section.
+
+    Note: tailor's Award is also populated from this section; this richer extra
+    file preserves everything (issuer, date, description) for the user.
+    """
+
+    id: str
+    title: str
+    issuer: str | None = None
+    date: str | None = None
+    summary: str | None = None

linkfetch/parse/__init__.py

@@ -0,0 +1,43 @@
+"""Per-section HTML parsers.
+
+Each section module exposes ``parse(html: str) -> list[Model]`` (or a single
+model for basics). Parsers are **pure**: lxml in, pydantic out, no browser, no
+network. They anchor on LinkedIn's *stable* structure — the ``<main>`` content
+``<section>``, its repeated entry blocks, and the ordered ``<p>``/``<span>`` text
+fragments inside each (see :mod:`linkfetch.parse.common`) — rather than the
+randomized hashed utility class names, so they survive cosmetic markup churn.
+"""
+
+from __future__ import annotations
+
+from linkfetch.parse import (
+    awards,
+    basics,
+    certifications,
+    courses,
+    education,
+    experience,
+    honors,
+    languages,
+    patents,
+    projects,
+    recommendations,
+    skills,
+    volunteering,
+)
+
+__all__ = [
+    "awards",
+    "basics",
+    "certifications",
+    "courses",
+    "education",
+    "experience",
+    "honors",
+    "languages",
+    "patents",
+    "projects",
+    "recommendations",
+    "skills",
+    "volunteering",
+]

linkfetch/parse/awards.py

@@ -0,0 +1,57 @@
+"""Parse Honors & awards into tailor's Award model.
+
+Ordered fragments per entry::
+
+    [title, "Issued by <issuer> · <date>", "Associated with <entity>", <description>]
+
+The issuer and date are pulled from the "Issued by … · …" fragment; the
+expandable description (when present) becomes the bullets. See honors.py for the
+richer extra-file view of the same section.
+"""
+
+from __future__ import annotations
+
+import re
+
+from linkfetch.models import Award
+from linkfetch.parse.common import description_text, entry_items, ordered_texts
+from linkfetch.text import clean, slugify, split_bullets
+
+_ISSUED_BY = re.compile(r"^issued by\s+", re.I)
+
+
+def split_issuer_date(frag: str) -> tuple[str | None, str | None]:
+    """``"Issued by Roche · Apr 2021"`` -> ``("Roche", "Apr 2021")``."""
+    body = _ISSUED_BY.sub("", frag)
+    issuer, sep, date = body.partition("·")
+    return clean(issuer), (clean(date) if sep else None)
+
+
+def extract(texts: list[str]) -> tuple[str, str | None, str | None]:
+    """Return ``(title, issuer, date)`` from an entry's fragments."""
+    title = texts[0]
+    issuer = date = None
+    for frag in texts[1:]:
+        if frag.lower().startswith("issued by"):
+            issuer, date = split_issuer_date(frag)
+            break
+    return title, issuer, date
+
+
+def parse(html: str) -> list[Award]:
+    out: list[Award] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        title, issuer, date = extract(texts)
+        out.append(
+            Award(
+                id=slugify(title, issuer),
+                name=title,
+                issuer=issuer,
+                date=date,
+                bullets=split_bullets(description_text(el)),
+            )
+        )
+    return out

linkfetch/parse/basics.py

@@ -0,0 +1,89 @@
+"""Parse the top profile card for basics (name, headline, location, about).
+
+Unlike the section details pages, basics come from the main profile page. The
+captured DOM has no ``<h1>`` or ``#about`` anchor, so we anchor on stable signals
+instead:
+
+* **name** — the page ``<title>`` ("Jane Doe | LinkedIn"), with the first
+  matching visible fragment as a fallback;
+* **headline** — the visible fragment immediately following the name;
+* **location** — the first short "City, Region, Country" fragment;
+* **about** — the single expandable text box on the page (the About summary).
+"""
+
+from __future__ import annotations
+
+import re
+
+from lxml import html as lhtml
+
+from linkfetch.models import Basics
+from linkfetch.parse.common import description_text, ordered_texts
+from linkfetch.text import clean
+
+# A "City, Region, Country" style location: 2-3 comma-separated short parts.
+_LOCATION = re.compile(r"^[^,\n]{2,40}(?:,[^,\n]{2,40}){1,2}$")
+
+
+def parse(html: str) -> Basics:
+    if not html or not html.strip():
+        return Basics(name="")
+    root = lhtml.fromstring(html)
+
+    name = _name(root)
+    fragments = ordered_texts(root)
+
+    headline = None
+    if name:
+        for i, frag in enumerate(fragments):
+            if frag == name and i + 1 < len(fragments):
+                headline = fragments[i + 1]
+                break
+
+    # Location: a "City, Region, Country" fragment in the top card, anchored by
+    # the "Contact info"/"connections" line that LinkedIn renders right after it.
+    location = None
+    for i, frag in enumerate(fragments):
+        if frag in (name, headline) or not _LOCATION.match(frag):
+            continue
+        if _looks_like_company(frag) or _is_location_junk(frag):
+            continue
+        nearby = " ".join(fragments[i + 1 : i + 4]).lower()
+        if "contact info" in nearby or "connection" in nearby:
+            location = frag
+            break
+
+    return Basics(
+        name=name or "",
+        label=headline,
+        location=location,
+        summary=description_text(root, exclude_linked=True),
+    )
+
+
+def _name(root: lhtml.HtmlElement) -> str | None:
+    titles = root.xpath("//title/text()")
+    if titles:
+        title = clean(titles[0]) or ""
+        # "Jane Doe | LinkedIn" / "… | (5) LinkedIn"
+        name = re.split(r"\s*[|–—]\s*", title)[0]
+        name = clean(name)
+        if name and name.lower() != "linkedin":
+            return name
+    h1 = root.xpath("//h1")
+    for h in h1:
+        txt = clean(h.text_content())
+        if txt:
+            return txt
+    return None
+
+
+def _looks_like_company(frag: str) -> bool:
+    low = frag.lower()
+    return "·" in frag or "|" in frag or "linkedin" in low
+
+
+def _is_location_junk(frag: str) -> bool:
+    """Reject media-player / dialog strings that happen to contain commas."""
+    low = frag.lower()
+    return any(w in low for w in ("live", "seek", "modal", "dialog", "opacity"))

linkfetch/parse/certifications.py

@@ -0,0 +1,70 @@
+"""Parse the Licenses & certifications details page.
+
+Ordered fragments per entry::
+
+    [name, issuer, "Issued <date>" (· "Expires <date>"), "Show credential",
+     "Credential ID …", "Skills: …"]
+
+The credential URL is the anchor whose visible text is "Show credential".
+"""
+
+from __future__ import annotations
+
+import re
+
+from linkfetch.models import Certification, Link
+from linkfetch.parse.common import entry_items, ordered_texts
+from linkfetch.text import clean, slugify
+
+
+def parse(html: str) -> list[Certification]:
+    out: list[Certification] = []
+    for el in entry_items(html):
+        texts = ordered_texts(el)
+        if not texts:
+            continue
+        name = texts[0]
+        issuer = texts[1] if len(texts) > 1 else None
+
+        issued = expires = credential_id = None
+        for frag in texts[2:]:
+            low = frag.lower()
+            if "issued" in low or "expires" in low or "expir" in low:
+                issued, expires = _split_dates(frag)
+            elif "credential id" in low:
+                credential_id = clean(frag.split(":", 1)[-1])
+
+        href = _credential_link(el)
+        links = [Link(label="Credential", url=href)] if href else []
+        out.append(
+            Certification(
+                id=slugify(name, issuer),
+                name=name,
+                issuer=issuer,
+                issued=issued,
+                expires=expires,
+                credential_id=credential_id,
+                links=links,
+            )
+        )
+    return out
+
+
+def _credential_link(el) -> str | None:
+    """Anchor whose text reads 'Show credential' (the external credential URL)."""
+    for a in el.xpath(".//a[@href]"):
+        text = clean(a.text_content()) or ""
+        if "credential" in text.lower():
+            return a.get("href").split("?")[0]
+    return None
+
+
+def _split_dates(text: str) -> tuple[str | None, str | None]:
+    issued = expires = None
+    m = re.search(r"issued\s+(.+?)(?:·|$)", text, re.I)
+    if m:
+        issued = clean(m.group(1))
+    m = re.search(r"expir\w*\s+(.+?)(?:·|$)", text, re.I)
+    if m:
+        expires = clean(m.group(1))
+    return issued, expires