LinkedInWebScraper 1.1.1__tar.gz

linkedinwebscraper-1.1.1/.dockerignore

@@ -0,0 +1,19 @@
+.git
+.gitignore
+.github
+.pytest_cache
+.ruff_cache
+.tox
+.venv
+.tmp
+artifacts
+build
+dist
+site
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+*.sqlite
+*.db
+*.log

linkedinwebscraper-1.1.1/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+ARG INSTALL_EXTRAS=""
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY LICENSE README.md MANIFEST.in pyproject.toml setup.py ./
+COPY src ./src
+COPY main.py process_ds_jobs.py runtime.example.toml ./
+
+RUN python -m pip install --upgrade pip \
+    && if [ -n "$INSTALL_EXTRAS" ]; then pip install ".[$INSTALL_EXTRAS]"; else pip install .; fi
+
+VOLUME ["/app/artifacts"]
+
+ENTRYPOINT ["linkedin-webscraper"]
+CMD ["scrape", "daily"]

linkedinwebscraper-1.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ricardo García Ramírez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

linkedinwebscraper-1.1.1/MANIFEST.in

@@ -0,0 +1,6 @@
+include README.md
+include LICENSE
+include runtime.example.toml
+include Dockerfile
+include .dockerignore
+recursive-include examples *.py

linkedinwebscraper-1.1.1/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: LinkedInWebScraper
+Version: 1.1.1
+Summary: A library for scraping LinkedIn job postings.
+Author-email: Ricardo Garcia Ramirez <rgr.5882@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ricardogr07/LinkedInWebScraper
+Project-URL: Source, https://github.com/ricardogr07/LinkedInWebScraper
+Keywords: linkedin,scraper,jobs,openai,pandas,sqlite
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: SQLAlchemy>=2.0.36
+Provides-Extra: openai
+Requires-Dist: openai>=2.29.0; extra == "openai"
+Requires-Dist: pydantic>=2.7.0; extra == "openai"
+Provides-Extra: dev
+Requires-Dist: build>=1.2.2; extra == "dev"
+Requires-Dist: coverage[toml]>=7.6.0; extra == "dev"
+Requires-Dist: mkdocs>=1.6.1; extra == "dev"
+Requires-Dist: mkdocs-material>=9.5.34; extra == "dev"
+Requires-Dist: mkdocstrings[python]>=0.26.1; extra == "dev"
+Requires-Dist: openai>=2.29.0; extra == "dev"
+Requires-Dist: pydantic>=2.7.0; extra == "dev"
+Requires-Dist: pytest>=8.3.2; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Requires-Dist: pyrefly>=0.26.0; extra == "dev"
+Requires-Dist: ruff>=0.6.9; extra == "dev"
+Requires-Dist: tox>=4.21.2; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.6.1; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5.34; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.26.1; extra == "docs"
+Dynamic: license-file
+
+# LinkedInWebScraper
+
+[![CI](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/ci.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/ci.yml)
+[![Docs](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/docs.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/docs.yml)
+[![Docs site](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://ricardogr07.github.io/LinkedInWebScraper/)
+[![Release](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/release.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/release.yml)
+[![PyPI version](https://img.shields.io/pypi/v/LinkedInWebScraper.svg)](https://pypi.org/project/LinkedInWebScraper/)
+[![Python versions](https://img.shields.io/pypi/pyversions/LinkedInWebScraper.svg)](https://pypi.org/project/LinkedInWebScraper/)
+[![License](https://img.shields.io/pypi/l/LinkedInWebScraper.svg)](https://github.com/ricardogr07/LinkedInWebScraper/blob/main/LICENSE)
+
+LinkedInWebScraper is a production-minded Python library and scheduled job runner for collecting LinkedIn job listings, normalizing the data, persisting run history, and exporting reusable datasets.
+
+## Highlights
+
+- Canonical package namespace under `linkedin_web_scraper`
+- Typed programmatic config for single scrapes and TOML runtime config for CLI and scheduled runs
+- Managed artifacts under `artifacts/jobs`, `artifacts/logs`, and `artifacts/state`
+- SQLite-backed persistence through a clean application storage port
+- Package CLI with `scrape once`, `scrape daily`, `export`, and `--dry-run`
+- Optional OpenAI enrichment built on the current Responses API
+- Runnable examples under `examples/`
+- Auto release automation that waits for green CI and Docs runs on `main`
+
+## Install
+
+```bash
+pip install LinkedInWebScraper
+pip install LinkedInWebScraper[openai]
+pip install -e .[dev]
+```
+
+## Quickstart
+
+```python
+from linkedin_web_scraper import (
+    JobScraperConfig,
+    LinkedInJobScraper,
+    RemoteType,
+    configure_logging,
+)
+
+logger = configure_logging(filename="example.log")
+config = JobScraperConfig(
+    position="Data Analyst",
+    location="San Francisco",
+    remote=RemoteType.REMOTE,
+)
+
+jobs = LinkedInJobScraper(logger=logger, config=config).run()
+print(jobs.head())
+```
+
+## Examples
+
+Run the example scripts from `examples/`:
+
+```bash
+python examples/example.py
+python examples/example_advanced_config.py
+python examples/example_openai.py
+```
+
+The OpenAI example requires `OPENAI_API_KEY` in the environment.
+
+## CLI Runtime
+
+```bash
+linkedin-webscraper scrape once --dry-run
+linkedin-webscraper scrape daily
+linkedin-webscraper export --run-id <run-id>
+```
+
+Use `runtime.example.toml` as the template for a real `runtime.toml`. The root runtime scripts remain available for the daily and once workflows:
+
+```bash
+python main.py
+python process_ds_jobs.py
+```
+
+## Docs
+
+- [Getting Started](docs/getting-started.md)
+- [Configuration](docs/configuration.md)
+- [Runtime and Deployment](docs/runtime.md)
+- [Release and Automation](docs/development/release-and-automation.md)
+- [Validation](docs/development/validation.md)
+- [API Reference](docs/api.md)
+
+## Development
+
+Run the local gate before risky pushes or merges:
+
+```bash
+python -m tox -e preflight
+```
+
+For a faster smoke-only path:
+
+```bash
+python -m tox -e smoke
+```
+
+The detailed validation matrix and release flow live in `docs/development/validation.md` and `docs/development/release-and-automation.md`.
+
+## License
+
+This project is licensed under the MIT License.

linkedinwebscraper-1.1.1/README.md

@@ -0,0 +1,107 @@
+# LinkedInWebScraper
+
+[![CI](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/ci.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/ci.yml)
+[![Docs](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/docs.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/docs.yml)
+[![Docs site](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://ricardogr07.github.io/LinkedInWebScraper/)
+[![Release](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/release.yml/badge.svg)](https://github.com/ricardogr07/LinkedInWebScraper/actions/workflows/release.yml)
+[![PyPI version](https://img.shields.io/pypi/v/LinkedInWebScraper.svg)](https://pypi.org/project/LinkedInWebScraper/)
+[![Python versions](https://img.shields.io/pypi/pyversions/LinkedInWebScraper.svg)](https://pypi.org/project/LinkedInWebScraper/)
+[![License](https://img.shields.io/pypi/l/LinkedInWebScraper.svg)](https://github.com/ricardogr07/LinkedInWebScraper/blob/main/LICENSE)
+
+LinkedInWebScraper is a production-minded Python library and scheduled job runner for collecting LinkedIn job listings, normalizing the data, persisting run history, and exporting reusable datasets.
+
+## Highlights
+
+- Canonical package namespace under `linkedin_web_scraper`
+- Typed programmatic config for single scrapes and TOML runtime config for CLI and scheduled runs
+- Managed artifacts under `artifacts/jobs`, `artifacts/logs`, and `artifacts/state`
+- SQLite-backed persistence through a clean application storage port
+- Package CLI with `scrape once`, `scrape daily`, `export`, and `--dry-run`
+- Optional OpenAI enrichment built on the current Responses API
+- Runnable examples under `examples/`
+- Auto release automation that waits for green CI and Docs runs on `main`
+
+## Install
+
+```bash
+pip install LinkedInWebScraper
+pip install LinkedInWebScraper[openai]
+pip install -e .[dev]
+```
+
+## Quickstart
+
+```python
+from linkedin_web_scraper import (
+    JobScraperConfig,
+    LinkedInJobScraper,
+    RemoteType,
+    configure_logging,
+)
+
+logger = configure_logging(filename="example.log")
+config = JobScraperConfig(
+    position="Data Analyst",
+    location="San Francisco",
+    remote=RemoteType.REMOTE,
+)
+
+jobs = LinkedInJobScraper(logger=logger, config=config).run()
+print(jobs.head())
+```
+
+## Examples
+
+Run the example scripts from `examples/`:
+
+```bash
+python examples/example.py
+python examples/example_advanced_config.py
+python examples/example_openai.py
+```
+
+The OpenAI example requires `OPENAI_API_KEY` in the environment.
+
+## CLI Runtime
+
+```bash
+linkedin-webscraper scrape once --dry-run
+linkedin-webscraper scrape daily
+linkedin-webscraper export --run-id <run-id>
+```
+
+Use `runtime.example.toml` as the template for a real `runtime.toml`. The root runtime scripts remain available for the daily and once workflows:
+
+```bash
+python main.py
+python process_ds_jobs.py
+```
+
+## Docs
+
+- [Getting Started](docs/getting-started.md)
+- [Configuration](docs/configuration.md)
+- [Runtime and Deployment](docs/runtime.md)
+- [Release and Automation](docs/development/release-and-automation.md)
+- [Validation](docs/development/validation.md)
+- [API Reference](docs/api.md)
+
+## Development
+
+Run the local gate before risky pushes or merges:
+
+```bash
+python -m tox -e preflight
+```
+
+For a faster smoke-only path:
+
+```bash
+python -m tox -e smoke
+```
+
+The detailed validation matrix and release flow live in `docs/development/validation.md` and `docs/development/release-and-automation.md`.
+
+## License
+
+This project is licensed under the MIT License.

linkedinwebscraper-1.1.1/examples/example.py

@@ -0,0 +1,15 @@
+from linkedin_web_scraper import JobScraperConfig, LinkedInJobScraper, Logger
+
+logger = Logger("example.log")
+
+# Define scraper configuration
+config = JobScraperConfig(position="Data Analyst", location="San Francisco", remote="REMOTE")
+
+# Initialize the scraper
+scraper = LinkedInJobScraper(logger, config)
+
+# Scrape job data
+job_data = scraper.run()
+
+# View the results
+print(job_data.head())

linkedinwebscraper-1.1.1/examples/example_advanced_config.py

@@ -0,0 +1,38 @@
+from linkedin_web_scraper import (
+    JobScraperAdvancedConfig,
+    JobScraperConfig,
+    LinkedInJobScraper,
+    Logger,
+)
+
+logger = Logger("example_advanced_config.log")
+
+KEYWORDS_LIST = [
+    "data",
+    "analytics",
+    "business intelligence",
+    "bi",
+    "statistical",
+    "statistics",
+    "analysis",
+    "power bi",
+]
+
+advanced_config = JobScraperAdvancedConfig(KEYWORDS=KEYWORDS_LIST)
+
+# Define scraper configuration
+config = JobScraperConfig(
+    position="Data Analyst",
+    location="San Francisco",
+    remote="REMOTE",
+    advanced_config=advanced_config,
+)
+
+# Initialize the scraper
+scraper = LinkedInJobScraper(logger, config)
+
+# Scrape job data
+job_data = scraper.run()
+
+# View the results
+print(job_data.head())

linkedinwebscraper-1.1.1/examples/example_openai.py

@@ -0,0 +1,21 @@
+from linkedin_web_scraper import JobScraperConfig, LinkedInJobScraper, Logger
+
+logger = Logger("example_openai.log")
+
+# Define scraper configuration
+config = JobScraperConfig(
+    position="Data Analyst",
+    location="San Francisco",
+    remote="REMOTE",
+    openai_enabled=True,
+    openai_model="gpt-4o-mini",
+)
+
+# Initialize the scraper
+scraper = LinkedInJobScraper(logger, config)
+
+# Scrape job data
+job_data = scraper.run()
+
+# View the results
+print(job_data.head())

linkedinwebscraper-1.1.1/pyproject.toml

@@ -0,0 +1,134 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "LinkedInWebScraper"
+version = "1.1.1"
+description = "A library for scraping LinkedIn job postings."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+  { name = "Ricardo Garcia Ramirez", email = "rgr.5882@gmail.com" },
+]
+keywords = ["linkedin", "scraper", "jobs", "openai", "pandas", "sqlite"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "beautifulsoup4>=4.12.3",
+  "numpy>=1.26.4",
+  "pandas>=2.2.0",
+  "requests>=2.31.0",
+  "SQLAlchemy>=2.0.36",
+]
+
+[project.optional-dependencies]
+openai = [
+  "openai>=2.29.0",
+  "pydantic>=2.7.0",
+]
+dev = [
+  "build>=1.2.2",
+  "coverage[toml]>=7.6.0",
+  "mkdocs>=1.6.1",
+  "mkdocs-material>=9.5.34",
+  "mkdocstrings[python]>=0.26.1",
+  "openai>=2.29.0",
+  "pydantic>=2.7.0",
+  "pytest>=8.3.2",
+  "pytest-cov>=5.0.0",
+  "pyrefly>=0.26.0",
+  "ruff>=0.6.9",
+  "tox>=4.21.2",
+]
+docs = [
+  "mkdocs>=1.6.1",
+  "mkdocs-material>=9.5.34",
+  "mkdocstrings[python]>=0.26.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/ricardogr07/LinkedInWebScraper"
+Source = "https://github.com/ricardogr07/LinkedInWebScraper"
+
+[project.scripts]
+linkedin-webscraper = "linkedin_web_scraper.interfaces.cli.main:main"
+
+[tool.setuptools]
+include-package-data = true
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["linkedin_web_scraper*"]
+
+[tool.pytest.ini_options]
+addopts = "-ra -p no:cacheprovider"
+testpaths = ["tests"]
+markers = [
+  "live_linkedin: tests that hit LinkedIn or related live HTTP endpoints",
+  "live_openai: tests that hit the OpenAI API",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = ["src", "."]
+extend-exclude = ["site", ".tmp", "pytest-cache-files-*"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I", "UP", "B", "SIM"]
+
+[tool.coverage.run]
+branch = true
+source = ["src"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = false
+
+[tool.pyrefly]
+project-includes = [
+  "src/linkedin_web_scraper/config/job_scraper_config.py",
+  "src/linkedin_web_scraper/config/job_scraper_advanced_config.py",
+  "src/linkedin_web_scraper/config/job_scraper_config_factory.py",
+  "src/linkedin_web_scraper/config/openai.py",
+  "src/linkedin_web_scraper/config/storage.py",
+  "src/linkedin_web_scraper/config/options.py",
+  "src/linkedin_web_scraper/config/runtime.py",
+  "src/linkedin_web_scraper/application/daily_scrape_service.py",
+  "src/linkedin_web_scraper/application/linkedin_job_scraper.py",
+  "src/linkedin_web_scraper/application/storage.py",
+  "src/linkedin_web_scraper/application/runtime_runner.py",
+  "src/linkedin_web_scraper/application/release_manager.py",
+  "src/linkedin_web_scraper/domain/job_data_cleaner.py",
+  "src/linkedin_web_scraper/domain/job_title_classifier.py",
+  "src/linkedin_web_scraper/infra/logging.py",
+  "src/linkedin_web_scraper/infra/paths.py",
+  "src/linkedin_web_scraper/infra/http/policy.py",
+  "src/linkedin_web_scraper/infra/http/utils.py",
+  "src/linkedin_web_scraper/infra/http/job_scraper.py",
+  "src/linkedin_web_scraper/infra/openai/models.py",
+  "src/linkedin_web_scraper/infra/openai/openai_handler.py",
+  "src/linkedin_web_scraper/infra/openai/job_description_processor.py",
+  "src/linkedin_web_scraper/infra/storage/models.py",
+  "src/linkedin_web_scraper/infra/storage/sqlite.py",
+]
+search-path = ["src"]
+python-version = "3.11"
+use-ignore-files = true

linkedinwebscraper-1.1.1/runtime.example.toml

@@ -0,0 +1,35 @@
+# Example runtime configuration for CLI, containers, and scheduled runs.
+# Keep OPENAI_API_KEY in the environment, never in this file.
+
+[logging]
+level = "INFO"
+file_name = "main.log"
+
+[storage]
+file_name = "linkedin_jobs.sqlite"
+state_dir = "artifacts/state"
+
+[scrape.once]
+position = "Data Scientist"
+location = "Monterrey"
+openai_enabled = false
+openai_model = "gpt-4o-mini"
+time_posted = "DAY"
+remote_types = ["REMOTE", "HYBRID", "ON-SITE"]
+file_name = "LinkedIn_Jobs_Data_Scientist_Monterrey.csv"
+output_dir = "artifacts/jobs"
+append = true
+
+[scrape.daily]
+cities = ["Monterrey", "Guadalajara", "Mexico City"]
+position = "Data Scientist"
+openai_enabled = false
+openai_model = "gpt-4o-mini"
+time_posted = "DAY"
+output_dir = "artifacts/jobs"
+combined_file_name = "LinkedIn_Jobs_Data_Scientist_Mexico.csv"
+
+[export]
+run_id = ""
+file_name = "linkedin_jobs_export.csv"
+output_dir = "artifacts/jobs"

linkedinwebscraper-1.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

linkedinwebscraper-1.1.1/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()