linkedin-pulse-search-posts 0.1.0__tar.gz

linkedin_pulse_search_posts-0.1.0/.gitignore

@@ -0,0 +1,39 @@
+cookies.json
+cookies*.json
+storage_state.json
+session_snapshot.json
+*.session
+linkedin-session*
+credentials.json
+secrets.json
+.secrets
+posts.json
+selected_posts.json
+worker/results/*.json
+.env
+.venv/
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+*.egg
+dist/
+build/
+**/dist/
+**/build/
+**/*.egg-info/
+.pytest_cache/
+
+# Cursor debug session logs
+**/.cursor/debug-*.log
+
+# Frontend (Next.js)
+frontend/node_modules/
+frontend/.next/
+frontend/.vercel/
+frontend/.env.local
+frontend/.env.*.local
+
+# Local tools (portable Node, etc.)
+.tools/

linkedin_pulse_search_posts-0.1.0/LICENSE

@@ -0,0 +1,3 @@
+Proprietary. All rights reserved.
+This software is licensed for use by the copyright holder only.
+Redistribution or publication without permission is not allowed.

linkedin_pulse_search_posts-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.5
+Name: linkedin-pulse-search-posts
+Version: 0.1.0
+Summary: LinkedIn post search built on linkedin-search-core.
+Project-URL: Homepage, https://github.com/vkudymov/linkedin-pulse-reader-v.3
+Project-URL: Repository, https://github.com/vkudymov/linkedin-pulse-reader-v.3
+Project-URL: Issues, https://github.com/vkudymov/linkedin-pulse-reader-v.3/issues
+Author: LinkedIn Pulse Reader
+Maintainer: LinkedIn Pulse Reader
+License: Proprietary
+License-File: LICENSE
+Keywords: linkedin,playwright,posts
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: linkedin-search-core>=0.1.0
+Requires-Dist: playwright>=1.49.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# linkedin-search-posts
+
+## Description
+
+LinkedIn feed post search. Depends on `linkedin-search-core` and does not import the jobs library.
+
+Requires Python 3.11+.
+
+## Installation
+
+```bash
+pip install linkedin-search-posts
+```
+
+That also installs `linkedin-search-core`.
+
+Local editable install (core first):
+
+```bash
+pip install -e packages/linkedin-search-core
+pip install -e packages/linkedin-search-posts
+```
+
+## Dependencies
+
+- `linkedin-search-core>=0.1.0`
+- `playwright>=1.49.0`
+
+## Usage
+
+```python
+from linkedin_search_core import LinkedInSession
+from linkedin_search_posts import PostSearch
+
+with LinkedInSession(cookies=cookies) as session:
+    posts = PostSearch(session).fetch(limit=10)
+```
+
+## API
+
+- `PostSearch.fetch(limit=10)` — open the feed, scroll, and parse posts
+- `PostSearch.read(limit=10)` — parse posts already on the current page
+- `PostSearch.search(limit=10)` — alias of `fetch`
+- `Author`, `Post`

linkedin_pulse_search_posts-0.1.0/README.md

@@ -0,0 +1,44 @@
+# linkedin-search-posts
+
+## Description
+
+LinkedIn feed post search. Depends on `linkedin-search-core` and does not import the jobs library.
+
+Requires Python 3.11+.
+
+## Installation
+
+```bash
+pip install linkedin-search-posts
+```
+
+That also installs `linkedin-search-core`.
+
+Local editable install (core first):
+
+```bash
+pip install -e packages/linkedin-search-core
+pip install -e packages/linkedin-search-posts
+```
+
+## Dependencies
+
+- `linkedin-search-core>=0.1.0`
+- `playwright>=1.49.0`
+
+## Usage
+
+```python
+from linkedin_search_core import LinkedInSession
+from linkedin_search_posts import PostSearch
+
+with LinkedInSession(cookies=cookies) as session:
+    posts = PostSearch(session).fetch(limit=10)
+```
+
+## API
+
+- `PostSearch.fetch(limit=10)` — open the feed, scroll, and parse posts
+- `PostSearch.read(limit=10)` — parse posts already on the current page
+- `PostSearch.search(limit=10)` — alias of `fetch`
+- `Author`, `Post`

linkedin_pulse_search_posts-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling>=1.21.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "linkedin-pulse-search-posts"
+version = "0.1.0"
+description = "LinkedIn post search built on linkedin-search-core."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "Proprietary" }
+authors = [{ name = "LinkedIn Pulse Reader" }]
+maintainers = [{ name = "LinkedIn Pulse Reader" }]
+keywords = ["linkedin", "posts", "playwright"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: Other/Proprietary License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Typing :: Typed",
+]
+dependencies = [
+  "linkedin-search-core>=0.1.0",
+  "playwright>=1.49.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/vkudymov/linkedin-pulse-reader-v.3"
+Repository = "https://github.com/vkudymov/linkedin-pulse-reader-v.3"
+Issues = "https://github.com/vkudymov/linkedin-pulse-reader-v.3/issues"
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0.0",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/src",
+  "/README.md",
+  "/LICENSE",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/linkedin_search_posts"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/__init__.py

@@ -0,0 +1,6 @@
+"""LinkedIn post search on top of linkedin-search-core."""
+
+from .models.post import Author, Post
+from .search import PostSearch
+
+__all__ = ["Author", "Post", "PostSearch"]

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/loading/__init__.py

@@ -0,0 +1,3 @@
+from .waiter import FeedWaiter
+
+__all__ = ["FeedWaiter"]

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/loading/waiter.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+"""
+RU: Ожидания и стабилизация динамической ленты.
+    Определяет “готовность” страницы для дальнейших шагов (скролл/парсинг) через наблюдаемые сигналы
+    DOM, чтобы снизить флейки в SPA.
+
+EN: Waiting and stabilization for a dynamic feed.
+    Defines “page readiness” for subsequent steps (scrolling/parsing) using observable DOM signals to
+    reduce SPA flakiness.
+"""
+
+import time
+from dataclasses import dataclass
+from typing import Literal
+
+from playwright.sync_api import Page, TimeoutError as PlaywrightTimeoutError
+
+from linkedin_search_core.exceptions import FeedLoadError
+
+
+@dataclass(frozen=True, slots=True)
+class FeedWaiter:
+    """
+    RU: Политика готовности feed.
+        Используется между навигацией и скроллом/парсингом; при проблемах поднимает `FeedLoadError`,
+        чтобы оркестратор мог выполнить retry/reload.
+
+    EN: Feed readiness policy.
+        Used between navigation and scrolling/parsing; raises `FeedLoadError` so the orchestrator can
+        retry/reload.
+    """
+
+    timeout_ms: int
+
+    feed_container_selectors: tuple[str, ...] = (
+        "main[role='main']",
+        "div[role='main']",
+        "div.application-outlet",
+        "div#react-root",
+        "div#content",
+        "div#main",
+        "main",
+        "div.scaffold-layout",
+        "div.scaffold-layout__content",
+        "div.scaffold-layout__inner",
+        "div.scaffold-layout__main",
+        "div.scaffold-finite-scroll",
+        "div.scaffold-finite-scroll__content",
+        # Last-resort: always present once DOM is there.
+        "body",
+    )
+    # post_container_selector_group: str = (
+    #     "div.feed-shared-update-v2, "
+    #     "div.occludable-update, "
+    #     "article[data-urn], "
+    #     "div[data-urn], "
+    #     "div[data-id], "
+    #     "div[data-urn*='urn:li:activity'], "
+    #     "div[data-urn*='urn:li:ugcPost'], "
+    #     "div[data-urn*='urn:li:share']"
+    # )
+    post_container_selector_group: str = "[data-testid='expandable-text-box']"
+    text_selector = "[data-testid='expandable-text-box']"
+    # Fallback anchors for accounts/variants where data-testid is absent.
+    text_selector_fallbacks: tuple[str, ...] = (
+        "div.update-components-text",
+        "div.feed-shared-update-v2__description",
+        "div.feed-shared-update-v2",
+        "div.occludable-update",
+        "article[data-urn]",
+        "div[data-urn*='urn:li:activity']",
+    )
+
+    def wait_for_feed_ready(self, page: Page) -> None:
+        deadline = time.time() + (self.timeout_ms / 1000.0)
+
+        # print(
+        #     "POST CONTAINERS:", self.page.locator("div.feed-shared-update-v2").count()
+        # )
+        # print(
+        #     "TEXT BLOCKS:",
+        #     self.page.locator("[data-testid='expandable-text-box']").count(),
+        # )
+
+        # 1) Ensure primary layout exists.
+        # Use state="attached" here: LinkedIn containers may exist but not be considered "visible"
+        # due to overlays/transitions; we only need the layout to be present before waiting for posts.
+        self._wait_for_any_selector(
+            page, self.feed_container_selectors, deadline, state="attached"
+        )
+
+        # 2) Best-effort: wait for at least one content anchor, but do not fail the whole flow.
+        # The selector-agnostic scroll loop can still make progress and parsing can be retried later.
+        remaining_ms = max(0, int((deadline - time.time()) * 1000))
+        try:
+            timeout_ms = min(remaining_ms, 6000)
+
+            selector_used: str | None = None
+            selectors = (self.text_selector, *self.text_selector_fallbacks)
+            start = time.time()
+            for sel in selectors:
+                budget_ms = max(
+                    1, int(timeout_ms * (1.0 - min(0.9, (time.time() - start) / 6.0)))
+                )
+                try:
+                    page.wait_for_selector(sel, timeout=budget_ms, state="attached")
+                    selector_used = sel
+                    break
+                except Exception:  # noqa: BLE001 - best-effort across fallbacks
+                    continue
+
+            if not selector_used:
+                raise PlaywrightTimeoutError("No feed content anchor found.")
+        except PlaywrightTimeoutError:
+            # Best-effort: do not fail the whole run here.
+            # The scroll loop + parsers can still make progress once LinkedIn finishes rendering.
+            return
+        # except PlaywrightTimeoutError:
+        #     return
+
+        # 3) Best-effort stabilization to reduce races while parsing.
+        try:
+            self._wait_for_stable_post_count(page, deadline)
+        except FeedLoadError:
+            return
+
+    def wait_for_new_posts(
+        self, page: Page, previous_count: int, timeout_ms: int
+    ) -> int:
+        deadline = time.time() + (timeout_ms / 1000.0)
+        locator = page.locator(self.post_container_selector_group)
+        while time.time() < deadline:
+            count = locator.count()
+            if count > previous_count:
+                return count
+            page.wait_for_timeout(250)
+        return locator.count()
+
+    def current_post_count(self, page: Page) -> int:
+        return page.locator(self.post_container_selector_group).count()
+
+    def _wait_for_any_selector(
+        self,
+        page: Page,
+        selectors: tuple[str, ...],
+        deadline: float,
+        *,
+        state: Literal["attached", "detached", "hidden", "visible"] = "visible",
+    ) -> None:
+        last_error: Exception | None = None
+        for idx, sel in enumerate(selectors):
+            remaining_ms = max(0, int((deadline - time.time()) * 1000))
+            if remaining_ms <= 0:
+                break
+            try:
+                # Budget remaining time across fallback selectors.
+                selectors_left = max(1, len(selectors) - idx)
+                used_timeout = max(1, int(remaining_ms / selectors_left))
+                page.wait_for_selector(sel, timeout=used_timeout, state=state)
+                return
+            except Exception as e:  # noqa: BLE001 - best-effort across fallbacks
+                last_error = e
+                continue
+        raise FeedLoadError(
+            "Timed out waiting for LinkedIn feed container to load."
+        ) from last_error
+
+    def _wait_for_stable_post_count(self, page: Page, deadline: float) -> None:
+        locator = page.locator(self.post_container_selector_group)
+        stable_samples = 0
+        last_count = -1
+        while time.time() < deadline:
+            count = locator.count()
+            if count == last_count and count > 0:
+                stable_samples += 1
+                if stable_samples >= 2:
+                    return
+            else:
+                stable_samples = 0
+                last_count = count
+            page.wait_for_timeout(350)
+        raise FeedLoadError("Feed did not stabilize in time (post list kept changing).")

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/models/__init__.py

@@ -0,0 +1,3 @@
+from .post import Author, Post
+
+__all__ = ["Author", "Post"]

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/models/post.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Any, Mapping
+
+
+@dataclass(frozen=True, slots=True)
+class Author:
+    """
+    RU: Доменная модель автора поста (boundary-объект).
+        Возвращается наружу как стабильный контракт, независимо от того, как меняется DOM LinkedIn.
+
+    EN: Post author domain model (boundary object).
+        Returned as a stable contract, independent of LinkedIn DOM changes.
+    """
+    name: str
+    headline: str | None = None
+    profile_url: str | None = None
+    urn: str | None = None
+    avatar_url: str | None = None
+
+    extra: Mapping[str, Any] = field(default_factory=dict, repr=False)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "extra", MappingProxyType(dict(self.extra)))
+
+
+def merge_authors(a: Author | None, b: Author | None) -> Author | None:
+    if a is None:
+        return b
+    if b is None:
+        return a
+    return Author(
+        name=a.name or b.name,
+        headline=a.headline or b.headline,
+        profile_url=a.profile_url or b.profile_url,
+        urn=a.urn or b.urn,
+        avatar_url=a.avatar_url or b.avatar_url,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class Post:
+    """
+    RU: Доменная модель поста из ленты (публичный контракт).
+        Поля optional по дизайну: LinkedIn UI/типы постов различаются, а парсер работает best-effort.
+
+    EN: Feed post domain model (public contract).
+        Optional fields are intentional: LinkedIn UI/post types vary and parsing is best-effort.
+    """
+
+    id: str | None = None
+    urn: str | None = None
+    post_url: str | None = None
+
+    author: Author | None = None
+    content: str | None = None
+    published_at_text: str | None = None
+
+    reactions_count: int | None = None
+    comments_count: int | None = None
+
+    media_urls: tuple[str, ...] = ()
+
+    extra: Mapping[str, Any] = field(default_factory=dict, repr=False)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "extra", MappingProxyType(dict(self.extra)))
+

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/navigation/__init__.py

@@ -0,0 +1,3 @@
+from .feed import FeedNavigator
+
+__all__ = ["FeedNavigator"]

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/navigation/feed.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+"""
+RU: Навигация к LinkedIn feed и детекция auth-редиректов.
+    Модуль отделяет “куда перейти” от ожиданий/скролла/парсинга и явно сигнализирует об отсутствии
+    авторизации через `LoginRequiredError`.
+
+EN: Navigation to LinkedIn feed and auth-redirect detection.
+    Separates “where to go” from waiting/scrolling/parsing and explicitly signals missing auth via
+    `LoginRequiredError`.
+"""
+
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import Any
+
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import Page, TimeoutError as PlaywrightTimeoutError
+
+from linkedin_search_core.exceptions import FeedLoadError, LoginRequiredError
+
+_session_is_logged_out: Any
+try:
+    from session_snapshot import is_logged_out as _session_is_logged_out
+except ImportError:  # pragma: no cover
+    _session_is_logged_out = None
+
+
+@dataclass(frozen=True, slots=True)
+class FeedNavigator:
+    """
+    RU: Роутер фичи “feed”.
+        Используется оркестратором перед ожиданиями/скроллом/парсингом, чтобы гарантировать корректную
+        целевую страницу.
+
+    EN: Feed feature router.
+        Used by the orchestrator before waiting/scrolling/parsing to ensure the correct target page.
+    """
+    feed_url: str = "https://www.linkedin.com/feed/"
+
+    def goto_feed(self, page: Page) -> None:
+        # If we're already on /feed, avoid re-navigation (LinkedIn can be slow/flaky on reload).
+        try:
+            current = (page.url or "").lower()
+        except Exception:
+            current = ""
+
+        if "linkedin.com/feed" not in current:
+            try:
+                page.goto(self.feed_url, wait_until="domcontentloaded")
+            except PlaywrightTimeoutError:
+                # Fallback: LinkedIn may keep the document "loading" for a long time.
+                # We only need the navigation to commit; readiness is handled by waiter/selectors.
+                try:
+                    page.goto(self.feed_url, wait_until="commit")
+                except PlaywrightError as e:
+                    raise FeedLoadError(_feed_open_error_message(e)) from e
+            except PlaywrightError as e:
+                raise FeedLoadError(_feed_open_error_message(e)) from e
+
+        # LinkedIn may serve authwall/login UI for /feed without changing the URL immediately.
+        # Wait briefly for either feed layout or login markers, then decide.
+        with suppress(Exception):
+            page.wait_for_selector(
+                "main[role='main'], div.application-outlet, "
+                "input#username, input[name='session_key'], input[name='session_password']",
+                timeout=10_000,
+            )
+
+        if self._looks_like_login_page(page) or self._looks_like_checkpoint_page(page):
+            raise LoginRequiredError(
+                "LinkedIn returned a login/authwall page for the feed URL. "
+                "Provide valid cookies or complete manual login first."
+            )
+        if self._is_auth_redirect(page.url):
+            raise LoginRequiredError(
+                "LinkedIn redirected to an authentication/checkpoint page. "
+                "Provide valid cookies or complete manual login first."
+            )
+
+    @staticmethod
+    def _is_auth_redirect(url: str) -> bool:
+        if _session_is_logged_out is not None:
+            return bool(_session_is_logged_out(url or ""))
+        lowered = (url or "").lower()
+        return any(
+            marker in lowered
+            for marker in (
+                "/login",
+                "/checkpoint",
+                "/authwall",
+                "linkedin.com/uas/",
+            )
+        )
+
+    @staticmethod
+    def _looks_like_login_page(page: Page) -> bool:
+        """
+        RU: LinkedIn может отдавать страницу логина даже на /feed (URL не всегда меняется).
+        EN: LinkedIn may serve a login page even on /feed (URL does not always change).
+        """
+        with suppress(Exception):
+            title = (page.title() or "").lower()
+            if "sign in" in title or "login" in title:
+                return True
+
+        try:
+            # Login form markers (avoid user content / PII).
+            return page.locator(
+                "input#username, input[name='session_key'], input[name='session_password']"
+            ).count() > 0
+        except Exception:
+            return False
+
+    @staticmethod
+    def _looks_like_checkpoint_page(page: Page) -> bool:
+        """
+        RU: Иногда LinkedIn показывает checkpoint/verification UI, при этом URL может оставаться /feed.
+        EN: LinkedIn may show checkpoint/verification UI while the URL still looks like /feed.
+        """
+        try:
+            return (
+                page.locator(
+                    "form[action*='checkpoint'], input[name='challengeId'], input[name='pin']"
+                ).count()
+                > 0
+            )
+        except Exception:
+            return False
+
+
+def _safe_url(url: str) -> str:
+    # Avoid logging full URLs with params/fragments.
+    try:
+        from urllib.parse import urlparse
+
+        p = urlparse(url)
+        return f"{p.scheme}://{p.netloc}{p.path}"
+    except Exception:
+        return url
+
+
+def _feed_open_error_message(error: Exception) -> str:
+    detail = str(error).splitlines()[0] if str(error).strip() else type(error).__name__
+    return (
+        "LinkedIn feed did not open: https://www.linkedin.com/feed/. "
+        "Check internet connection, VPN/proxy, and that LinkedIn opens in a regular browser. "
+        f"Playwright error: {detail}"
+    )
+
+
+def _safe_title(page: Page) -> str:
+    try:
+        t = page.title()
+        return t[:120]
+    except Exception:
+        return ""
+

linkedin_pulse_search_posts-0.1.0/src/linkedin_search_posts/parsing/__init__.py

@@ -0,0 +1,3 @@
+from .post_parser import PostParser
+
+__all__ = ["PostParser"]