afrojobspy 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- afrojobspy-0.1.0/PKG-INFO +8 -0
- afrojobspy-0.1.0/README.md +28 -0
- afrojobspy-0.1.0/setup.cfg +4 -0
- afrojobspy-0.1.0/setup.py +14 -0
- afrojobspy-0.1.0/src/afrojobspy/__init__.py +74 -0
- afrojobspy-0.1.0/src/afrojobspy/models.py +13 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/__init__.py +6 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/addisjobs.py +66 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/afriwork.py +115 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/aujobs.py +79 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/base.py +27 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/brightermonday.py +81 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/careerjunction.py +77 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/careers24.py +77 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/dereja.py +66 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/ethiojobs.py +84 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/geezjobs.py +64 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/hahujobs.py +81 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/jobberman.py +118 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/jobsethio.py +66 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/myjobmag.py +101 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/reliefweb.py +79 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/unjobs.py +57 -0
- afrojobspy-0.1.0/src/afrojobspy/scrapers/wuzzuf.py +75 -0
- afrojobspy-0.1.0/src/afrojobspy.egg-info/PKG-INFO +8 -0
- afrojobspy-0.1.0/src/afrojobspy.egg-info/SOURCES.txt +27 -0
- afrojobspy-0.1.0/src/afrojobspy.egg-info/dependency_links.txt +1 -0
- afrojobspy-0.1.0/src/afrojobspy.egg-info/requires.txt +4 -0
- afrojobspy-0.1.0/src/afrojobspy.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# AfroJobsPy 🌍💼
|
|
2
|
+
|
|
3
|
+
AfroJobsPy is a lightweight and modular Python library designed to scrape job listings from leading African employment platforms.
|
|
4
|
+
|
|
5
|
+
## Supported Job Boards
|
|
6
|
+
* **EthioJobs** (`ethiojobs`)
|
|
7
|
+
* **Fuzu** (`fuzu`)
|
|
8
|
+
* **GeezJobs** (`geezjobs`)
|
|
9
|
+
* **Jobberman** (`jobberman`)
|
|
10
|
+
* **JobWebZambia** (`jobwebzambia`)
|
|
11
|
+
* **MyJobMag** (`myjobmag`)
|
|
12
|
+
* **Rekrute** (`rekrute`)
|
|
13
|
+
* **TunisiaTravail** (`tunisiatravail`)
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
Clone the repository and install it locally in editable mode:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install -e .
|
|
21
|
+
|
|
22
|
+
from afrojobspy.scrapers.ethiojobs import EthioJobsScraper
|
|
23
|
+
|
|
24
|
+
scraper = EthioJobsScraper()
|
|
25
|
+
jobs = scraper.scrape(search_term="Developer", results_wanted=5)
|
|
26
|
+
|
|
27
|
+
for job in jobs:
|
|
28
|
+
print(f"{job.title} at {job.company} -> {job.job_url}")
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name="afrojobspy",
|
|
5
|
+
version="0.1.0",
|
|
6
|
+
packages=find_packages(where="src"),
|
|
7
|
+
package_dir={"": "src"},
|
|
8
|
+
install_requires=[
|
|
9
|
+
"requests>=2.31.0",
|
|
10
|
+
"beautifulsoup4>=4.12.0",
|
|
11
|
+
"pydantic>=2.0.0",
|
|
12
|
+
"pandas>=2.0.0",
|
|
13
|
+
],
|
|
14
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
|
|
5
|
+
from afrojobspy.models import JobPost
|
|
6
|
+
from afrojobspy.scrapers.ethiojobs import EthiojobsScraper
|
|
7
|
+
from afrojobspy.scrapers.afriwork import AfriworkScraper
|
|
8
|
+
from afrojobspy.scrapers.aujobs import AUJobsScraper
|
|
9
|
+
from afrojobspy.scrapers.brightermonday import BrighterMondayScraper
|
|
10
|
+
from afrojobspy.scrapers.hahujobs import HahuJobsScraper
|
|
11
|
+
from afrojobspy.scrapers.geezjobs import GeezJobsScraper
|
|
12
|
+
from afrojobspy.models import JobPost
|
|
13
|
+
from afrojobspy.scrapers.ethiojobs import EthiojobsScraper
|
|
14
|
+
from afrojobspy.scrapers.geezjobs import GeezJobsScraper
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
# Map site keys to their corresponding scraper classes
|
|
21
|
+
SCRAPER_REGISTRY = {
|
|
22
|
+
"ethiojobs": EthiojobsScraper,
|
|
23
|
+
"afriwork": AfriworkScraper,
|
|
24
|
+
"au_jobs": AUJobsScraper,
|
|
25
|
+
"brightermonday": BrighterMondayScraper,
|
|
26
|
+
"hahujobs": HahuJobsScraper,
|
|
27
|
+
"geezjobs": GeezJobsScraper,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
def scrape_jobs(
|
|
31
|
+
search_term: str = "",
|
|
32
|
+
location: str = "",
|
|
33
|
+
results_wanted: int = 10,
|
|
34
|
+
sites: Optional[List[str]] = None,
|
|
35
|
+
proxies: Optional[List[str]] = None
|
|
36
|
+
) -> pd.DataFrame:
|
|
37
|
+
"""
|
|
38
|
+
Orchestrates job scraping across multiple African job platforms
|
|
39
|
+
and returns a consolidated pandas DataFrame.
|
|
40
|
+
"""
|
|
41
|
+
if sites is None:
|
|
42
|
+
sites = list(SCRAPER_REGISTRY.keys())
|
|
43
|
+
|
|
44
|
+
all_jobs: List[JobPost] = []
|
|
45
|
+
|
|
46
|
+
for site_key in sites:
|
|
47
|
+
site_key = site_key.lower().strip()
|
|
48
|
+
if site_key not in SCRAPER_REGISTRY:
|
|
49
|
+
logger.warning(f"Unknown or unregistered site key: '{site_key}'. Skipping.")
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
scraper_cls = SCRAPER_REGISTRY[site_key]
|
|
53
|
+
try:
|
|
54
|
+
logger.info(f"Initializing scraper for {site_key}...")
|
|
55
|
+
scraper = scraper_cls(proxies=proxies)
|
|
56
|
+
jobs = scraper.scrape(search_term=search_term, location=location, results_wanted=results_wanted)
|
|
57
|
+
logger.info(f"Successfully scraped {len(jobs)} jobs from {site_key}.")
|
|
58
|
+
all_jobs.extend(jobs)
|
|
59
|
+
except Exception as e:
|
|
60
|
+
logger.error(f"Error scraping {site_key}: {e}")
|
|
61
|
+
|
|
62
|
+
# Convert JobPost dataclass/pydantic models to a list of dicts for pandas
|
|
63
|
+
jobs_data = [job.model_dump() if hasattr(job, "model_dump") else job.__dict__ for job in all_jobs]
|
|
64
|
+
|
|
65
|
+
if not jobs_data:
|
|
66
|
+
return pd.DataFrame(columns=["id", "title", "company", "location", "site_name", "job_url", "description", "date_posted"])
|
|
67
|
+
|
|
68
|
+
df = pd.DataFrame(jobs_data)
|
|
69
|
+
|
|
70
|
+
# Drop exact duplicates if any URL repeats
|
|
71
|
+
if "job_url" in df.columns:
|
|
72
|
+
df = df.drop_duplicates(subset=["job_url"])
|
|
73
|
+
|
|
74
|
+
return df
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
from pydantic import BaseModel, Field
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
class JobPost(BaseModel):
|
|
6
|
+
id: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
7
|
+
title: str
|
|
8
|
+
company: Optional[str] = "N/A"
|
|
9
|
+
location: Optional[str] = "Ethiopia"
|
|
10
|
+
site_name: str
|
|
11
|
+
job_url: str
|
|
12
|
+
description: Optional[str] = None
|
|
13
|
+
date_posted: Optional[str] = None
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
2
|
+
from afrojobspy.scrapers.myjobmag import MyJobMagScraper
|
|
3
|
+
from afrojobspy.scrapers.jobberman import JobbermanScraper
|
|
4
|
+
from afrojobspy.scrapers.ethiojobs import EthiojobsScraper
|
|
5
|
+
from afrojobspy.scrapers.reliefweb import ReliefWebScraper
|
|
6
|
+
from afrojobspy.scrapers.unjobs import UNJobsScraper
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, Optional
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
5
|
+
from afrojobspy.models import JobPost
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
class AddisJobsScraper(BaseScraper):
|
|
10
|
+
"""Scraper for Addis Jobs"""
|
|
11
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
12
|
+
super().__init__(proxies=proxies)
|
|
13
|
+
self.base_url = "https://addisjobs.net"
|
|
14
|
+
self.session.headers.update({
|
|
15
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
16
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
def scrape(self, search_term: str = "", location: str = "", results_wanted: int = 10) -> List[JobPost]:
|
|
20
|
+
jobs = []
|
|
21
|
+
try:
|
|
22
|
+
response = self.session.get(self.base_url, timeout=15)
|
|
23
|
+
if response.status_code != 200:
|
|
24
|
+
return jobs
|
|
25
|
+
|
|
26
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
|
27
|
+
cards = soup.select("div.job-card, article, div.item, a[href*='/job']")
|
|
28
|
+
|
|
29
|
+
seen_urls = set()
|
|
30
|
+
for card in cards:
|
|
31
|
+
link = card if card.name == "a" else card.find("a", href=True)
|
|
32
|
+
if not link:
|
|
33
|
+
continue
|
|
34
|
+
|
|
35
|
+
job_url = link.get("href", "")
|
|
36
|
+
title = link.get_text(strip=True)
|
|
37
|
+
|
|
38
|
+
if not job_url or not title or len(title) < 5:
|
|
39
|
+
continue
|
|
40
|
+
if any(skip in title.lower() for skip in ["home", "login", "register", "about", "contact", "privacy", "terms"]):
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
if not job_url.startswith("http"):
|
|
44
|
+
job_url = "https://addisjobs.net" + job_url if job_url.startswith("/") else f"https://addisjobs.net/{job_url}"
|
|
45
|
+
|
|
46
|
+
job_url = job_url.split("?")[0]
|
|
47
|
+
if job_url in seen_urls:
|
|
48
|
+
continue
|
|
49
|
+
seen_urls.add(job_url)
|
|
50
|
+
|
|
51
|
+
if search_term and search_term.lower() not in title.lower():
|
|
52
|
+
continue
|
|
53
|
+
|
|
54
|
+
jobs.append(JobPost(
|
|
55
|
+
title=title,
|
|
56
|
+
company="Addis Jobs Employer",
|
|
57
|
+
location=location or "Addis Ababa, Ethiopia",
|
|
58
|
+
site_name="addisjobs",
|
|
59
|
+
job_url=job_url
|
|
60
|
+
))
|
|
61
|
+
|
|
62
|
+
if len(jobs) >= results_wanted:
|
|
63
|
+
break
|
|
64
|
+
except Exception as e:
|
|
65
|
+
logger.error(f"Addis Jobs scrape error: {e}")
|
|
66
|
+
return jobs
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
from bs4 import BeautifulSoup
|
|
5
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
6
|
+
from afrojobspy.models import JobPost
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
class AfriworkScraper(BaseScraper):
|
|
11
|
+
"""Scraper for Afriwork Ethiopia"""
|
|
12
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
13
|
+
super().__init__(proxies=proxies)
|
|
14
|
+
self.base_url = "https://afriworket.com/jobs"
|
|
15
|
+
self.session.headers.update({
|
|
16
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
17
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
def scrape(self, search_term: str = "", location: str = "", results_wanted: int = 10) -> List[JobPost]:
|
|
21
|
+
jobs = []
|
|
22
|
+
try:
|
|
23
|
+
response = self.session.get(self.base_url, timeout=15)
|
|
24
|
+
if response.status_code != 200:
|
|
25
|
+
return jobs
|
|
26
|
+
|
|
27
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
|
28
|
+
|
|
29
|
+
# 1. Try extracting Next.js hydration data if present
|
|
30
|
+
script_tag = soup.find("script", id="__NEXT_DATA__")
|
|
31
|
+
if script_tag and script_tag.string:
|
|
32
|
+
try:
|
|
33
|
+
data = json.loads(script_tag.string)
|
|
34
|
+
page_props = data.get("props", {}).get("pageProps", {})
|
|
35
|
+
|
|
36
|
+
job_list = (
|
|
37
|
+
page_props.get("jobs", []) or
|
|
38
|
+
page_props.get("listings", []) or
|
|
39
|
+
page_props.get("vacancies", [])
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
if not job_list:
|
|
43
|
+
for k, v in page_props.items():
|
|
44
|
+
if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
|
|
45
|
+
if any(field in v[0] for field in ["title", "job_title", "position"]):
|
|
46
|
+
job_list = v
|
|
47
|
+
break
|
|
48
|
+
|
|
49
|
+
for item in job_list[:results_wanted]:
|
|
50
|
+
title = item.get("title") or item.get("job_title") or item.get("position", "")
|
|
51
|
+
if not title:
|
|
52
|
+
continue
|
|
53
|
+
|
|
54
|
+
job_id = item.get("id") or item.get("_id", "")
|
|
55
|
+
job_url = item.get("url") or (f"https://afriworket.com/jobs/{job_id}" if job_id else self.base_url)
|
|
56
|
+
|
|
57
|
+
company_obj = item.get("company")
|
|
58
|
+
company = company_obj.get("name") if isinstance(company_obj, dict) else (company_obj or "Afriwork Partner")
|
|
59
|
+
|
|
60
|
+
if search_term and search_term.lower() not in title.lower():
|
|
61
|
+
continue
|
|
62
|
+
|
|
63
|
+
jobs.append(JobPost(
|
|
64
|
+
title=title,
|
|
65
|
+
company=company,
|
|
66
|
+
location=location or item.get("location", "Addis Ababa, Ethiopia"),
|
|
67
|
+
site_name="afriwork",
|
|
68
|
+
job_url=job_url
|
|
69
|
+
))
|
|
70
|
+
except Exception as json_err:
|
|
71
|
+
logger.debug(f"Afriwork JSON hydration fallback triggered: {json_err}")
|
|
72
|
+
|
|
73
|
+
# 2. Fallback to general HTML anchor element parsing
|
|
74
|
+
if not jobs:
|
|
75
|
+
cards = soup.select("a[href*='/jobs/'], div.job-card, article")
|
|
76
|
+
seen_urls = set()
|
|
77
|
+
|
|
78
|
+
for card in cards:
|
|
79
|
+
link = card if card.name == "a" else card.find("a")
|
|
80
|
+
if not link:
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
job_url = link.get("href", "")
|
|
84
|
+
title = link.get_text(strip=True)
|
|
85
|
+
|
|
86
|
+
if not title or len(title) < 4 or "/jobs/" not in job_url:
|
|
87
|
+
continue
|
|
88
|
+
if job_url.rstrip("/") in ["/jobs", "https://afriworket.com/jobs"]:
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
if not job_url.startswith("http"):
|
|
92
|
+
job_url = "https://afriworket.com" + job_url if job_url.startswith("/") else f"https://afriworket.com/{job_url}"
|
|
93
|
+
|
|
94
|
+
job_url = job_url.split("?")[0]
|
|
95
|
+
|
|
96
|
+
if job_url in seen_urls:
|
|
97
|
+
continue
|
|
98
|
+
seen_urls.add(job_url)
|
|
99
|
+
|
|
100
|
+
if search_term and search_term.lower() not in title.lower():
|
|
101
|
+
continue
|
|
102
|
+
|
|
103
|
+
jobs.append(JobPost(
|
|
104
|
+
title=title,
|
|
105
|
+
company="Afriwork Partner",
|
|
106
|
+
location=location or "Addis Ababa, Ethiopia",
|
|
107
|
+
site_name="afriwork",
|
|
108
|
+
job_url=job_url
|
|
109
|
+
))
|
|
110
|
+
|
|
111
|
+
if len(jobs) >= results_wanted:
|
|
112
|
+
break
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logger.error(f"Afriwork scrape error: {e}")
|
|
115
|
+
return jobs
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, Optional
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
5
|
+
from afrojobspy.models import JobPost
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
class AUJobsScraper(BaseScraper):
|
|
10
|
+
"""Scraper for African Union (AU) Jobs Portal"""
|
|
11
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
12
|
+
super().__init__(proxies=proxies)
|
|
13
|
+
self.base_url = "https://jobs.au.int/search/?q=&searchResultView=LIST"
|
|
14
|
+
self.session.headers.update({
|
|
15
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
16
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
def scrape(self, search_term: str = "", location: str = "", results_wanted: int = 10) -> List[JobPost]:
|
|
20
|
+
jobs = []
|
|
21
|
+
try:
|
|
22
|
+
response = self.session.get(self.base_url, timeout=15)
|
|
23
|
+
if response.status_code != 200:
|
|
24
|
+
return jobs
|
|
25
|
+
|
|
26
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
|
27
|
+
|
|
28
|
+
# Target listing rows or table items on the AU jobs search page
|
|
29
|
+
cards = soup.select("tr.data-row, div.job-tile, .searchResults-row, tr[id^='row'], li.job-item, a[href*='/job/']")
|
|
30
|
+
|
|
31
|
+
seen_urls = set()
|
|
32
|
+
for card in cards:
|
|
33
|
+
link = card if card.name == "a" else card.find("a", href=True)
|
|
34
|
+
if not link:
|
|
35
|
+
continue
|
|
36
|
+
|
|
37
|
+
job_url = link.get("href", "")
|
|
38
|
+
title = link.get_text(strip=True)
|
|
39
|
+
|
|
40
|
+
if not job_url or not title or len(title) < 5:
|
|
41
|
+
continue
|
|
42
|
+
if "/job/" not in job_url:
|
|
43
|
+
continue
|
|
44
|
+
|
|
45
|
+
if any(skip in title.lower() for skip in ["home", "login", "register", "about", "contact", "privacy", "terms", "faq", "search"]):
|
|
46
|
+
continue
|
|
47
|
+
|
|
48
|
+
if not job_url.startswith("http"):
|
|
49
|
+
job_url = "https://jobs.au.int" + job_url if job_url.startswith("/") else f"https://jobs.au.int/{job_url}"
|
|
50
|
+
|
|
51
|
+
job_url = job_url.split("?")[0]
|
|
52
|
+
|
|
53
|
+
if job_url in seen_urls:
|
|
54
|
+
continue
|
|
55
|
+
seen_urls.add(job_url)
|
|
56
|
+
|
|
57
|
+
if search_term and search_term.lower() not in title.lower():
|
|
58
|
+
continue
|
|
59
|
+
|
|
60
|
+
job_location = location or "Addis Ababa, Ethiopia"
|
|
61
|
+
loc_elem = card.select_one(".jobLocation, .location, span.col")
|
|
62
|
+
if loc_elem:
|
|
63
|
+
loc_text = loc_elem.get_text(strip=True)
|
|
64
|
+
if loc_text and len(loc_text) < 40:
|
|
65
|
+
job_location = loc_text
|
|
66
|
+
|
|
67
|
+
jobs.append(JobPost(
|
|
68
|
+
title=title,
|
|
69
|
+
company="African Union (AU)",
|
|
70
|
+
location=job_location,
|
|
71
|
+
site_name="au_jobs",
|
|
72
|
+
job_url=job_url
|
|
73
|
+
))
|
|
74
|
+
|
|
75
|
+
if len(jobs) >= results_wanted:
|
|
76
|
+
break
|
|
77
|
+
except Exception as e:
|
|
78
|
+
logger.error(f"AU Jobs scrape error: {e}")
|
|
79
|
+
return jobs
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
from afrojobspy.models import JobPost
|
|
5
|
+
|
|
6
|
+
class BaseScraper(ABC):
|
|
7
|
+
"""
|
|
8
|
+
Abstract Base Class for all African job board scrapers.
|
|
9
|
+
Handles session initialization, default headers, and proxies.
|
|
10
|
+
"""
|
|
11
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
12
|
+
self.session = requests.Session()
|
|
13
|
+
# Spoof a modern browser User-Agent to avoid immediate blocks
|
|
14
|
+
self.session.headers.update({
|
|
15
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
16
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
17
|
+
"Chrome/122.0.0.0 Safari/537.36",
|
|
18
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
19
|
+
})
|
|
20
|
+
self.proxies = proxies
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def scrape(self, search_term: str, location: str, results_wanted: int) -> List[JobPost]:
|
|
24
|
+
"""
|
|
25
|
+
Every child scraper must implement this method to parse its specific site.
|
|
26
|
+
"""
|
|
27
|
+
pass
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, Optional
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
5
|
+
from afrojobspy.models import JobPost
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
class BrighterMondayScraper(BaseScraper):
|
|
10
|
+
"""Scraper for BrighterMonday East Africa"""
|
|
11
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
12
|
+
super().__init__(proxies=proxies)
|
|
13
|
+
self.base_url = "https://www.brightermonday.co.ke/jobs"
|
|
14
|
+
self.session.headers.update({
|
|
15
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
16
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
def scrape(self, search_term: str = "", location: str = "", results_wanted: int = 10) -> List[JobPost]:
|
|
20
|
+
jobs = []
|
|
21
|
+
try:
|
|
22
|
+
response = self.session.get(self.base_url, timeout=15)
|
|
23
|
+
if response.status_code != 200:
|
|
24
|
+
return jobs
|
|
25
|
+
|
|
26
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
|
27
|
+
|
|
28
|
+
# BrighterMonday list cards
|
|
29
|
+
cards = soup.select("div.panel, div.search-result-item, div.job-card, article, div.flex.flex-col")
|
|
30
|
+
|
|
31
|
+
seen_urls = set()
|
|
32
|
+
for card in cards:
|
|
33
|
+
link = card.find("a", href=True)
|
|
34
|
+
if not link:
|
|
35
|
+
continue
|
|
36
|
+
|
|
37
|
+
job_url = link.get("href", "")
|
|
38
|
+
|
|
39
|
+
# BrighterMonday active postings use the /listings/ path format
|
|
40
|
+
if not job_url or "/listings/" not in job_url:
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
title_elem = card.select_one("h3, h2, a.text-loading, .job-title")
|
|
44
|
+
title = title_elem.get_text(strip=True) if title_elem else link.get_text(strip=True)
|
|
45
|
+
|
|
46
|
+
if not title or len(title) < 4:
|
|
47
|
+
continue
|
|
48
|
+
|
|
49
|
+
if not job_url.startswith("http"):
|
|
50
|
+
job_url = "https://www.brightermonday.co.ke" + job_url if job_url.startswith("/") else f"https://www.brightermonday.co.ke/{job_url}"
|
|
51
|
+
|
|
52
|
+
job_url = job_url.split("?")[0]
|
|
53
|
+
|
|
54
|
+
if job_url in seen_urls:
|
|
55
|
+
continue
|
|
56
|
+
seen_urls.add(job_url)
|
|
57
|
+
|
|
58
|
+
if search_term and search_term.lower() not in title.lower():
|
|
59
|
+
continue
|
|
60
|
+
|
|
61
|
+
# Extract company name
|
|
62
|
+
company = "BrighterMonday Employer"
|
|
63
|
+
comp_elem = card.select_one(".text-gray-500, .job-company, span.text-sm")
|
|
64
|
+
if comp_elem:
|
|
65
|
+
c_text = comp_elem.get_text(strip=True)
|
|
66
|
+
if c_text and len(c_text) < 50 and "ago" not in c_text.lower():
|
|
67
|
+
company = c_text
|
|
68
|
+
|
|
69
|
+
jobs.append(JobPost(
|
|
70
|
+
title=title,
|
|
71
|
+
company=company,
|
|
72
|
+
location=location or "Kenya",
|
|
73
|
+
site_name="brightermonday",
|
|
74
|
+
job_url=job_url
|
|
75
|
+
))
|
|
76
|
+
|
|
77
|
+
if len(jobs) >= results_wanted:
|
|
78
|
+
break
|
|
79
|
+
except Exception as e:
|
|
80
|
+
logger.error(f"BrighterMonday scrape error: {e}")
|
|
81
|
+
return jobs
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, Optional
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from afrojobspy.scrapers.base import BaseScraper
|
|
5
|
+
from afrojobspy.models import JobPost
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
class CareerJunctionScraper(BaseScraper):
|
|
10
|
+
"""Scraper for CareerJunction (South Africa)"""
|
|
11
|
+
def __init__(self, proxies: Optional[List[str]] = None):
|
|
12
|
+
super().__init__(proxies=proxies)
|
|
13
|
+
self.base_url = "https://www.careerjunction.co.za/jobs"
|
|
14
|
+
self.session.headers.update({
|
|
15
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
16
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
def scrape(self, search_term: str = "", location: str = "", results_wanted: int = 10) -> List[JobPost]:
|
|
20
|
+
jobs = []
|
|
21
|
+
try:
|
|
22
|
+
response = self.session.get(self.base_url, timeout=15)
|
|
23
|
+
if response.status_code != 200:
|
|
24
|
+
return jobs
|
|
25
|
+
|
|
26
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
|
27
|
+
cards = soup.select("div.job-card, div.module-job, article, div.results-item")
|
|
28
|
+
|
|
29
|
+
seen_urls = set()
|
|
30
|
+
for card in cards:
|
|
31
|
+
link = card.find("a", href=True)
|
|
32
|
+
if not link:
|
|
33
|
+
continue
|
|
34
|
+
|
|
35
|
+
job_url = link.get("href", "")
|
|
36
|
+
if not job_url or "/jobs/" not in job_url:
|
|
37
|
+
continue
|
|
38
|
+
|
|
39
|
+
title_elem = card.select_one("h2, h3, .job-title, a")
|
|
40
|
+
title = title_elem.get_text(strip=True) if title_elem else link.get_text(strip=True)
|
|
41
|
+
|
|
42
|
+
if not title or len(title) < 4:
|
|
43
|
+
continue
|
|
44
|
+
if any(skip in title.lower() for skip in ["home", "login", "register", "about", "contact", "privacy", "terms"]):
|
|
45
|
+
continue
|
|
46
|
+
|
|
47
|
+
if not job_url.startswith("http"):
|
|
48
|
+
job_url = "https://www.careerjunction.co.za" + job_url if job_url.startswith("/") else f"https://www.careerjunction.co.za/{job_url}"
|
|
49
|
+
|
|
50
|
+
job_url = job_url.split("?")[0]
|
|
51
|
+
if job_url in seen_urls:
|
|
52
|
+
continue
|
|
53
|
+
seen_urls.add(job_url)
|
|
54
|
+
|
|
55
|
+
if search_term and search_term.lower() not in title.lower():
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
company = "CareerJunction Partner"
|
|
59
|
+
comp_elem = card.select_one(".company-name, .job-company")
|
|
60
|
+
if comp_elem:
|
|
61
|
+
c_text = comp_elem.get_text(strip=True)
|
|
62
|
+
if c_text and len(c_text) < 50:
|
|
63
|
+
company = c_text
|
|
64
|
+
|
|
65
|
+
jobs.append(JobPost(
|
|
66
|
+
title=title,
|
|
67
|
+
company=company,
|
|
68
|
+
location=location or "South Africa",
|
|
69
|
+
site_name="careerjunction",
|
|
70
|
+
job_url=job_url
|
|
71
|
+
))
|
|
72
|
+
|
|
73
|
+
if len(jobs) >= results_wanted:
|
|
74
|
+
break
|
|
75
|
+
except Exception as e:
|
|
76
|
+
logger.error(f"CareerJunction scrape error: {e}")
|
|
77
|
+
return jobs
|