python-jobspy-damarowen 1.2.0__py3-none-any.whl

jobspy/__init__.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Tuple
+
+import pandas as pd
+
+from jobspy.bayt import BaytScraper
+from jobspy.bdjobs import BDJobs
+from jobspy.glassdoor import Glassdoor
+from jobspy.google import Google
+from jobspy.indeed import Indeed
+from jobspy.linkedin import LinkedIn
+from jobspy.naukri import Naukri
+from jobspy.model import JobType, Location, JobResponse, Country
+from jobspy.model import SalarySource, ScraperInput, Site
+from jobspy.util import (
+    set_logger_level,
+    extract_salary,
+    create_logger,
+    get_enum_from_value,
+    map_str_to_site,
+    convert_to_annual,
+    desired_order,
+)
+from jobspy.ziprecruiter import ZipRecruiter
+from jobspy.jobstreet import JobStreet
+
+
+# Update the SCRAPER_MAPPING dictionary in the scrape_jobs function
+
+def scrape_jobs(
+    site_name: str | list[str] | Site | list[Site] | None = None,
+    search_term: str | None = None,
+    google_search_term: str | None = None,
+    location: str | None = None,
+    distance: int | None = 50,
+    is_remote: bool = False,
+    job_type: str | None = None,
+    easy_apply: bool | None = None,
+    results_wanted: int = 15,
+    country_indeed: str = "usa",
+    proxies: list[str] | str | None = None,
+    ca_cert: str | None = None,
+    description_format: str = "markdown",
+    linkedin_fetch_description: bool | None = False,
+    linkedin_company_ids: list[int] | None = None,
+    offset: int | None = 0,
+    hours_old: int = None,
+    enforce_annual_salary: bool = False,
+    verbose: int = 0,
+    user_agent: str = None,
+    **kwargs,
+) -> pd.DataFrame:
+    """
+    Scrapes job data from job boards concurrently
+    :return: Pandas DataFrame containing job data
+    """
+    SCRAPER_MAPPING = {
+        Site.LINKEDIN: LinkedIn,
+        Site.INDEED: Indeed,
+        Site.ZIP_RECRUITER: ZipRecruiter,
+        Site.GLASSDOOR: Glassdoor,
+        Site.GOOGLE: Google,
+        Site.BAYT: BaytScraper,
+        Site.NAUKRI: Naukri,
+        Site.BDJOBS: BDJobs,
+        Site.JOBSTREET: JobStreet,
+    }
+    set_logger_level(verbose)
+    job_type = get_enum_from_value(job_type) if job_type else None
+
+    def get_site_type():
+        site_types = list(Site)
+        if isinstance(site_name, str):
+            site_types = [map_str_to_site(site_name)]
+        elif isinstance(site_name, Site):
+            site_types = [site_name]
+        elif isinstance(site_name, list):
+            site_types = [
+                map_str_to_site(site) if isinstance(site, str) else site
+                for site in site_name
+            ]
+        return site_types
+
+    country_enum = Country.from_string(country_indeed)
+
+    scraper_input = ScraperInput(
+        site_type=get_site_type(),
+        country=country_enum,
+        search_term=search_term,
+        google_search_term=google_search_term,
+        location=location,
+        distance=distance,
+        is_remote=is_remote,
+        job_type=job_type,
+        easy_apply=easy_apply,
+        description_format=description_format,
+        linkedin_fetch_description=linkedin_fetch_description,
+        results_wanted=results_wanted,
+        linkedin_company_ids=linkedin_company_ids,
+        offset=offset,
+        hours_old=hours_old,
+    )
+
+    def scrape_site(site: Site) -> Tuple[str, JobResponse]:
+        scraper_class = SCRAPER_MAPPING[site]
+        scraper = scraper_class(proxies=proxies, ca_cert=ca_cert, user_agent=user_agent)
+        scraped_data: JobResponse = scraper.scrape(scraper_input)
+        cap_name = site.value.capitalize()
+        site_name = "ZipRecruiter" if cap_name == "Zip_recruiter" else cap_name
+        site_name = "LinkedIn" if cap_name == "Linkedin" else cap_name
+        create_logger(site_name).info(f"finished scraping")
+        return site.value, scraped_data
+
+    site_to_jobs_dict = {}
+
+    def worker(site):
+        site_val, scraped_info = scrape_site(site)
+        return site_val, scraped_info
+
+    with ThreadPoolExecutor() as executor:
+        future_to_site = {
+            executor.submit(worker, site): site for site in scraper_input.site_type
+        }
+
+        for future in as_completed(future_to_site):
+            site_value, scraped_data = future.result()
+            site_to_jobs_dict[site_value] = scraped_data
+
+    jobs_dfs: list[pd.DataFrame] = []
+
+    for site, job_response in site_to_jobs_dict.items():
+        for job in job_response.jobs:
+            job_data = job.dict()
+            job_url = job_data["job_url"]
+            job_data["site"] = site
+            job_data["company"] = job_data["company_name"]
+            job_data["job_type"] = (
+                ", ".join(job_type.value[0] for job_type in job_data["job_type"])
+                if job_data["job_type"]
+                else None
+            )
+            job_data["emails"] = (
+                ", ".join(job_data["emails"]) if job_data["emails"] else None
+            )
+            if job_data["location"]:
+                job_data["location"] = Location(
+                    **job_data["location"]
+                ).display_location()
+
+            # Handle compensation
+            compensation_obj = job_data.get("compensation")
+            if compensation_obj and isinstance(compensation_obj, dict):
+                job_data["interval"] = (
+                    compensation_obj.get("interval").value
+                    if compensation_obj.get("interval")
+                    else None
+                )
+                job_data["min_amount"] = compensation_obj.get("min_amount")
+                job_data["max_amount"] = compensation_obj.get("max_amount")
+                job_data["currency"] = compensation_obj.get("currency", "USD")
+                job_data["salary_source"] = SalarySource.DIRECT_DATA.value
+                if enforce_annual_salary and (
+                    job_data["interval"]
+                    and job_data["interval"] != "yearly"
+                    and job_data["min_amount"]
+                    and job_data["max_amount"]
+                ):
+                    convert_to_annual(job_data)
+            else:
+                if country_enum == Country.USA:
+                    (
+                        job_data["interval"],
+                        job_data["min_amount"],
+                        job_data["max_amount"],
+                        job_data["currency"],
+                    ) = extract_salary(
+                        job_data["description"],
+                        enforce_annual_salary=enforce_annual_salary,
+                    )
+                    job_data["salary_source"] = SalarySource.DESCRIPTION.value
+
+            job_data["salary_source"] = (
+                job_data["salary_source"]
+                if "min_amount" in job_data and job_data["min_amount"]
+                else None
+            )
+
+            #naukri-specific fields
+            job_data["skills"] = (
+                ", ".join(job_data["skills"]) if job_data["skills"] else None
+            )
+            job_data["experience_range"] = job_data.get("experience_range")
+            job_data["company_rating"] = job_data.get("company_rating")
+            job_data["company_reviews_count"] = job_data.get("company_reviews_count")
+            job_data["vacancy_count"] = job_data.get("vacancy_count")
+            job_data["work_from_home_type"] = job_data.get("work_from_home_type")
+
+            job_df = pd.DataFrame([job_data])
+            jobs_dfs.append(job_df)
+
+    if jobs_dfs:
+        # Step 1: Filter out all-NA columns from each DataFrame before concatenation
+        filtered_dfs = [df.dropna(axis=1, how="all") for df in jobs_dfs]
+
+        # Step 2: Concatenate the filtered DataFrames
+        jobs_df = pd.concat(filtered_dfs, ignore_index=True)
+
+        # Step 3: Ensure all desired columns are present, adding missing ones as empty
+        for column in desired_order:
+            if column not in jobs_df.columns:
+                jobs_df[column] = None  # Add missing columns as empty
+
+        # Reorder the DataFrame according to the desired order
+        jobs_df = jobs_df[desired_order]
+
+        # Step 4: Sort the DataFrame as required
+        return jobs_df.sort_values(
+            by=["site", "date_posted"], ascending=[True, False]
+        ).reset_index(drop=True)
+    else:
+        return pd.DataFrame()
+
+
+# Add BDJobs to __all__
+__all__ = [
+    "BDJobs",
+]

jobspy/bayt/__init__.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import random
+import time
+
+from bs4 import BeautifulSoup
+
+from jobspy.model import (
+    Scraper,
+    ScraperInput,
+    Site,
+    JobPost,
+    JobResponse,
+    Location,
+    Country,
+)
+from jobspy.util import create_logger, create_session
+
+log = create_logger("Bayt")
+
+
+class BaytScraper(Scraper):
+    base_url = "https://www.bayt.com"
+    delay = 2
+    band_delay = 3
+
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
+    ):
+        super().__init__(Site.BAYT, proxies=proxies, ca_cert=ca_cert)
+        self.scraper_input = None
+        self.session = None
+        self.country = "worldwide"
+
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
+        self.scraper_input = scraper_input
+        self.session = create_session(
+            proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True
+        )
+        job_list: list[JobPost] = []
+        page = 1
+        results_wanted = (
+            scraper_input.results_wanted if scraper_input.results_wanted else 10
+        )
+
+        while len(job_list) < results_wanted:
+            log.info(f"Fetching Bayt jobs page {page}")
+            job_elements = self._fetch_jobs(self.scraper_input.search_term, page)
+            if not job_elements:
+                break
+
+            if job_elements:
+                log.debug(
+                    "First job element snippet:\n" + job_elements[0].prettify()[:500]
+                )
+
+            initial_count = len(job_list)
+            for job in job_elements:
+                try:
+                    job_post = self._extract_job_info(job)
+                    if job_post:
+                        job_list.append(job_post)
+                        if len(job_list) >= results_wanted:
+                            break
+                    else:
+                        log.debug(
+                            "Extraction returned None. Job snippet:\n"
+                            + job.prettify()[:500]
+                        )
+                except Exception as e:
+                    log.error(f"Bayt: Error extracting job info: {str(e)}")
+                    continue
+
+            if len(job_list) == initial_count:
+                log.info(f"No new jobs found on page {page}. Ending pagination.")
+                break
+
+            page += 1
+            time.sleep(random.uniform(self.delay, self.delay + self.band_delay))
+
+        job_list = job_list[: scraper_input.results_wanted]
+        return JobResponse(jobs=job_list)
+
+    def _fetch_jobs(self, query: str, page: int) -> list | None:
+        """
+        Grabs the job results for the given query and page number.
+        """
+        try:
+            url = f"{self.base_url}/en/international/jobs/{query}-jobs/?page={page}"
+            response = self.session.get(url)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, "html.parser")
+            job_listings = soup.find_all("li", attrs={"data-js-job": ""})
+            log.debug(f"Found {len(job_listings)} job listing elements")
+            return job_listings
+        except Exception as e:
+            log.error(f"Bayt: Error fetching jobs - {str(e)}")
+            return None
+
+    def _extract_job_info(self, job: BeautifulSoup) -> JobPost | None:
+        """
+        Extracts the job information from a single job listing.
+        """
+        # Find the h2 element holding the title and link (no class filtering)
+        job_general_information = job.find("h2")
+        if not job_general_information:
+            return
+
+        job_title = job_general_information.get_text(strip=True)
+        job_url = self._extract_job_url(job_general_information)
+        if not job_url:
+            return
+
+        # Extract company name using the original approach:
+        company_tag = job.find("div", class_="t-nowrap p10l")
+        company_name = (
+            company_tag.find("span").get_text(strip=True)
+            if company_tag and company_tag.find("span")
+            else None
+        )
+
+        # Extract location using the original approach:
+        location_tag = job.find("div", class_="t-mute t-small")
+        location = location_tag.get_text(strip=True) if location_tag else None
+
+        job_id = f"bayt-{abs(hash(job_url))}"
+        location_obj = Location(
+            city=location,
+            country=Country.from_string(self.country),
+        )
+        return JobPost(
+            id=job_id,
+            title=job_title,
+            company_name=company_name,
+            location=location_obj,
+            job_url=job_url,
+        )
+
+    def _extract_job_url(self, job_general_information: BeautifulSoup) -> str | None:
+        """
+        Pulls the job URL from the 'a' within the h2 element.
+        """
+        a_tag = job_general_information.find("a")
+        if a_tag and a_tag.has_attr("href"):
+            return self.base_url + a_tag["href"].strip()