linkedin-playwright-scraper 4.0.0__tar.gz

linkedin_playwright_scraper-4.0.0/.env.example

@@ -0,0 +1,8 @@
+# LinkedIn credentials for scraping
+# Copy this file to .env and fill in your credentials
+
+# Use either LINKEDIN_EMAIL or LINKEDIN_USERNAME (both work)
+LINKEDIN_EMAIL=your.email@example.com
+# LINKEDIN_USERNAME=your.email@example.com
+
+LINKEDIN_PASSWORD=your_password_here

linkedin_playwright_scraper-4.0.0/.github/workflows/publish.yml

@@ -0,0 +1,25 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # required for PyPI trusted publishing (OIDC)
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Build distribution
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

linkedin_playwright_scraper-4.0.0/.gitignore

@@ -0,0 +1,45 @@
+*.swp
+linkedin_user_scraper.egg-info
+linkedin_scraper.egg-info
+dist
+build
+__pycache__
+*.pyc
+chromedriver
+.DS_Store
+.vscode/
+.env/
+scrape.py
+creds.json
+venv
+*.zip
+.env
+
+# Test outputs
+*.log
+test_*.db
+test_linkedin.db
+test_summary.json
+results_*.json
+person_*.json
+*_improved.json
+
+# Debug scripts (keep debug_connection_selectors.py)
+debug_*.py
+!debug_connection_selectors.py
+
+# Backup files
+*_old.py
+*.backup
+
+# Session files (sensitive cookies)
+linkedin_session.json
+
+# Build artifacts
+MANIFEST
+MANIFEST.ini
+.pytest_cache/
+
+# Basic package build artifacts
+build-basic/
+dist-basic/

linkedin_playwright_scraper-4.0.0/CLAUDE.md

@@ -0,0 +1,86 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+```bash
+# Install all dependencies and Playwright browsers
+just install
+
+# Run unit tests only (no LinkedIn session required)
+just test
+
+# Run a single test file
+just test-file tests/test_feed_scraper.py
+
+# Run a single test by name
+just test-one tests/test_feed_scraper.py TestFeedScraperUnit::test_parse_count
+
+# Run integration tests (requires linkedin_session.json)
+just test-integration
+
+# Create a LinkedIn session file (manual login in browser)
+just session
+
+# Format, lint, typecheck + unit tests
+just check
+```
+
+Toolchain: **uv** manages the virtualenv (`.venv/`) and dependencies (`pyproject.toml`). **just** is the task runner (`Justfile`). Run `just` with no arguments to list all recipes.
+
+## Architecture
+
+This is an async Playwright-based LinkedIn scraper. All scraping is fully async (asyncio).
+
+**Entry point:** `linkedin_scraper/__init__.py` re-exports everything public.
+
+### Layer structure
+
+```
+core/           Browser lifecycle, auth, utils, exceptions
+scrapers/       One scraper class per LinkedIn surface
+models/         Pydantic models for scraped data
+callbacks.py    Progress reporting (SilentCallback, ConsoleCallback, JSONLogCallback, MultiCallback)
+```
+
+### `core/`
+
+- `browser.py` — `BrowserManager`: async context manager wrapping Playwright. Handles launch, session save/load (`linkedin_session.json` = Playwright storage state), cookie injection.
+- `auth.py` — `login_with_credentials`, `login_with_cookie`, `is_logged_in`, `wait_for_manual_login`. Login detection is URL-based (not DOM) to survive LinkedIn A/B tests.
+- `utils.py` — `retry_async` decorator, `detect_rate_limit`, scroll helpers, `click_see_more_buttons`, `extract_text_safe`.
+- `exceptions.py` — exception hierarchy rooted at `LinkedInScraperException`.
+
+### `scrapers/`
+
+All scrapers inherit `BaseScraper` (`scrapers/base.py`) which wraps `core/` helpers as instance methods.
+
+| Scraper | Surface |
+|---|---|
+| `PersonScraper` | Profile page |
+| `CompanyScraper` | Company overview |
+| `CompanyPostsScraper` | Company `/posts/` tab |
+| `FeedScraper` | `/feed/` (authenticated) |
+| `JobScraper` | Individual job posting |
+| `JobSearchScraper` | Job search results |
+
+**Scraper pattern:** `scrape(url, limit)` → navigate → wait for content → scroll loop with deduplication by URN → return list of Pydantic models.
+
+Post content is extracted via `page.evaluate()` (inline JS) rather than Playwright locators, because LinkedIn's CSS class names are obfuscated and unstable. Activity URNs (`urn:li:activity:XXXXXX`) are the stable anchor — each post element has `data-urn` set to its URN.
+
+### `models/`
+
+All models are Pydantic `BaseModel`. `Post` is shared between `CompanyPostsScraper` and `FeedScraper`:
+
+```python
+Post: linkedin_url, urn, author_name, author_url, text, posted_date,
+      reactions_count, comments_count, reposts_count, image_urls, video_url, article_url
+```
+
+## Key conventions
+
+- **Session file:** `linkedin_session.json` at the project root — Playwright storage state (cookies + localStorage). Never commit. Generated by `samples/create_session.py`.
+- **headless=False** is required for LinkedIn — headless browsers get blocked. Integration tests and samples all use `headless=False`.
+- **Feed scraper requires auth:** `FeedScraper.scrape()` calls `ensure_logged_in()` before navigating. Other scrapers do not enforce this.
+- **Noise filtering in FeedScraper:** Sponsored/promoted posts are filtered out in the JS evaluation block by checking for `"Promoted"` / `"Sponsorisé"` labels and `[data-control-name="promoted"]`.
+- **Test markers:** `unit` (no session), `integration` (needs `linkedin_session.json`), `slow`. Unit tests mock the Playwright `Page` object.

linkedin_playwright_scraper-4.0.0/CONTRIBUTING.md

@@ -0,0 +1,206 @@
+# Contributing to LinkedIn Scraper
+
+Thank you for your interest in contributing to LinkedIn Scraper! This document provides guidelines for contributing to the project.
+
+## Getting Started
+
+### Prerequisites
+
+- Python 3.8 or higher
+- pip package manager
+- Git
+
+### Setting Up Development Environment
+
+1. **Clone the repository**
+   ```bash
+   git clone https://github.com/joeyism/linkedin_scraper.git
+   cd linkedin_scraper
+   ```
+
+2. **Create a virtual environment**
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   ```
+
+3. **Install dependencies**
+   ```bash
+   pip install -r requirements.txt
+   pip install -r requirements-dev.txt
+   ```
+
+4. **Install Playwright browsers**
+   ```bash
+   playwright install chromium
+   ```
+
+5. **Set up environment variables** (for testing)
+   ```bash
+   cp .env.example .env
+   # Edit .env and add your LinkedIn credentials (optional, for integration tests)
+   ```
+
+## Development Workflow
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest
+
+# Run specific test file
+pytest tests/test_person.py
+
+# Run with verbose output
+pytest -v
+
+# Run with coverage
+pytest --cov=linkedin_scraper
+```
+
+### Code Style
+
+This project follows these guidelines:
+
+- **PEP 8**: Python code style guide
+- **Type hints**: Use type annotations where appropriate
+- **Docstrings**: Document all public functions and classes
+- **Line length**: Maximum 100 characters
+
+Before submitting, ensure your code passes linting:
+
+```bash
+# Format code
+black linkedin_scraper/
+
+# Check for issues
+flake8 linkedin_scraper/
+mypy linkedin_scraper/
+```
+
+### Testing Your Changes
+
+1. Write tests for new functionality
+2. Ensure all existing tests pass
+3. Test manually with the sample scripts in `samples/`
+4. Verify documentation is updated
+
+## Making Changes
+
+### Branching Strategy
+
+- `main` - Stable release branch
+- `feature/your-feature` - New features
+- `fix/your-bugfix` - Bug fixes
+- `docs/your-doc-change` - Documentation updates
+
+### Commit Messages
+
+Write clear, descriptive commit messages:
+
+```
+Add support for scraping job descriptions
+
+- Extract full job description text
+- Parse job requirements section
+- Add tests for job description parsing
+```
+
+Format:
+- First line: Brief summary (50 chars or less)
+- Blank line
+- Detailed explanation (wrap at 72 chars)
+- List specific changes with bullet points
+
+### Pull Request Process
+
+1. **Create a new branch**
+   ```bash
+   git checkout -b feature/your-feature-name
+   ```
+
+2. **Make your changes**
+   - Write clean, well-documented code
+   - Add tests for new functionality
+   - Update documentation as needed
+
+3. **Commit your changes**
+   ```bash
+   git add .
+   git commit -m "Your descriptive commit message"
+   ```
+
+4. **Push to your fork**
+   ```bash
+   git push origin feature/your-feature-name
+   ```
+
+5. **Create a Pull Request**
+   - Go to the GitHub repository
+   - Click "New Pull Request"
+   - Select your branch
+   - Fill out the PR template
+   - Wait for review
+
+### Pull Request Guidelines
+
+- **Description**: Clearly describe what changes you made and why
+- **Tests**: Include tests that cover your changes
+- **Documentation**: Update README.md or other docs if needed
+- **Small PRs**: Keep changes focused and manageable
+- **Responsive**: Be ready to address feedback
+
+## What to Contribute
+
+### Good First Issues
+
+Look for issues labeled `good first issue` for beginner-friendly tasks:
+
+- Documentation improvements
+- Bug fixes
+- Additional test coverage
+- Code refactoring
+
+### Feature Requests
+
+Before implementing major features:
+
+1. Check existing issues to avoid duplication
+2. Open an issue to discuss the feature
+3. Wait for maintainer approval
+4. Implement the feature once approved
+
+### Bug Reports
+
+When reporting bugs, include:
+
+- Python version
+- Operating system
+- Steps to reproduce
+- Expected vs actual behavior
+- Error messages/stack traces
+- Sample code if possible
+
+## Code Review Process
+
+1. A maintainer will review your PR
+2. They may request changes
+3. Make requested changes and push updates
+4. Once approved, a maintainer will merge your PR
+
+## Questions?
+
+If you have questions about contributing:
+
+- Open an issue on GitHub
+- Check existing issues and discussions
+- Review the README.md for usage examples
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the Apache License 2.0.
+
+---
+
+Thank you for contributing to LinkedIn Scraper! Your efforts help make this project better for everyone.

linkedin_playwright_scraper-4.0.0/FORK_CONTEXT.md

@@ -0,0 +1,73 @@
+# Contexte du fork — linkedin_scraper
+
+## Origin
+
+Fork de [joeyism/linkedin_scraper](https://github.com/joeyism/linkedin_scraper).
+
+## Pourquoi ce fork
+
+### Problème de départ
+
+Ce projet est né d'un constat simple : **l'API officielle LinkedIn ne permet pas de lire les posts**.
+
+Un premier projet (`linkedin-mcp`, fork de FilippTrigub) expose des outils MCP pour interagir avec LinkedIn via l'API officielle. Il fonctionne pour :
+- S'authentifier via OAuth2
+- Créer des posts (`POST /v2/ugcPosts`)
+
+Mais la **lecture des posts est bloquée** par LinkedIn :
+
+| Scope | Usage | Disponibilité |
+|-------|-------|---------------|
+| `w_member_social` | Créer des posts | Apps standard ✅ |
+| `r_member_social` | Lire des posts | Marketing Developer Platform uniquement ❌ |
+
+Le scope `r_member_social` est réservé aux partenaires LinkedIn (Marketing Developer Platform). Il est inaccessible aux développeurs individuels. Même avec un code correct, l'API renvoie **403 Forbidden** sur tous les endpoints de lecture (`/v2/ugcPosts`, `/v2/shares`).
+
+### Solution retenue : scraping web avec Playwright
+
+L'API officielle étant une impasse, la seule approche réaliste est d'automatiser un navigateur authentifié. Ce repo utilise déjà Playwright et expose exactement la structure nécessaire.
+
+## Objectif de ce fork
+
+**Ajouter un `FeedScraper`** capable de récupérer les N premiers posts du feed LinkedIn de l'utilisateur authentifié.
+
+Le feed LinkedIn correspond à la page `linkedin.com/feed/` — ce que l'utilisateur voit quand il se connecte : posts de ses connexions, articles partagés, etc.
+
+### Ce qui existe déjà dans le repo
+
+- `CompanyPostsScraper` — scrape les posts d'une page entreprise
+- `BrowserManager` — gestion de session Playwright réutilisable
+- Modèle `Post` — structure de données pour un post
+
+### Ce qu'il faut ajouter
+
+Un `FeedScraper` calqué sur `CompanyPostsScraper` qui :
+1. Navigue sur `https://www.linkedin.com/feed/`
+2. Scrolle N fois pour charger les posts
+3. Parse le DOM pour extraire : auteur, texte, date, reactions, comments
+4. Filtre le bruit : publicités, suggestions "Vous connaissez peut-être...", posts sponsorisés
+5. Retourne une liste de `Post`
+
+## Complexité anticipée
+
+### Identique à CompanyPostsScraper
+- Session Playwright réutilisée
+- Infinite scroll avec `page.evaluate("window.scrollBy(...)")`
+
+### Spécifique au feed
+- **Bruit dans le DOM** : le feed mélange posts réels, pubs, suggestions → filtrage nécessaire
+- **Sélecteurs CSS instables** : LinkedIn obfusque ses class names, ils peuvent changer
+- **Anti-bot** : LinkedIn est légèrement plus vigilant sur le feed que sur les pages publiques
+
+## Prochaines étapes
+
+1. Lire le code de `CompanyPostsScraper` pour comprendre le pattern exact
+2. Inspecter le DOM du feed LinkedIn pour identifier les sélecteurs stables
+3. Implémenter `FeedScraper` sur le même modèle
+4. Tester avec différents comptes et volumes de posts
+
+## Liens
+
+- Repo original : https://github.com/joeyism/linkedin_scraper
+- Ce fork : https://github.com/vinzlac/linkedin_scraper
+- Projet MCP LinkedIn associé : https://github.com/vinzlac/linkedin-mcp (contexte origine)

linkedin_playwright_scraper-4.0.0/Justfile

@@ -0,0 +1,87 @@
+# linkedin_scraper — task runner
+# Requires: uv, just
+
+# Default: list available recipes
+default:
+    @just --list
+
+# Install all dependencies (including dev) and Playwright browsers
+install:
+    uv sync --group dev
+    uv run playwright install chromium
+
+# Run unit tests only (no LinkedIn session required)
+test:
+    uv run pytest -m "not integration and not slow"
+
+# Run a single test file
+test-file FILE:
+    uv run pytest {{ FILE }} -m "not integration"
+
+# Run a single test by name
+test-one FILE TEST:
+    uv run pytest {{ FILE }}::{{ TEST }}
+
+# Run integration tests (requires linkedin_session.json)
+test-integration:
+    uv run pytest -m integration
+
+# Run all tests with coverage report
+test-cov:
+    uv run pytest -m "not integration" --cov=linkedin_scraper --cov-report=term-missing
+
+# Format code with black
+fmt:
+    uv run black linkedin_scraper tests samples
+
+# Check formatting without modifying files
+fmt-check:
+    uv run black --check linkedin_scraper tests samples
+
+# Lint with flake8
+lint:
+    uv run flake8 linkedin_scraper tests
+
+# Type check with mypy
+typecheck:
+    uv run mypy linkedin_scraper
+
+# Run all checks (fmt-check + lint + typecheck + test)
+check: fmt-check lint test
+
+# Create a LinkedIn session file (opens browser for manual login)
+session:
+    uv run python samples/create_session.py
+
+# Scrape N posts from your LinkedIn feed (default: 10)
+run-feed N="10":
+    uv run python samples/scrape_feed.py {{ N }}
+
+# Scrape N posts from feed using a virtual display (for Linux servers without a GUI)
+# Requires: sudo apt install xvfb
+run-feed-xvfb N="10":
+    xvfb-run --server-args="-screen 0 1280x720x24" uv run python samples/scrape_feed.py {{ N }}
+
+# Debug DOM structure of LinkedIn feed
+debug-feed:
+    uv run python samples/debug_feed.py
+
+# Scrape a LinkedIn profile (URL or slug)
+run-person PROFILE:
+    uv run python samples/scrape_person.py {{ PROFILE }}
+
+# Debug DOM selectors on a profile page (helps fix broken selectors)
+debug-person PROFILE:
+    uv run python samples/debug_person.py {{ PROFILE }}
+
+# Run the company posts scraper sample
+run-company URL="https://www.linkedin.com/company/microsoft/":
+    uv run python samples/scrape_company_posts.py
+
+# Build distribution packages
+build:
+    uv build
+
+# Publish to PyPI (requires TWINE_USERNAME / TWINE_PASSWORD or ~/.pypirc)
+publish:
+    uv run twine upload dist/*