divapply 0.4.2__py3-none-any.whl

divapply/discovery/jobspy.py

@@ -0,0 +1,557 @@
+"""JobSpy-based job discovery: searches Indeed, LinkedIn, Glassdoor, ZipRecruiter.
+
+Uses python-jobspy to scrape multiple job boards, deduplicates results,
+parses salary ranges, and stores everything in the DivApply database.
+
+Search queries, locations, and filtering rules are loaded from the user's
+search configuration YAML (searches.yaml) rather than being hardcoded.
+"""
+
+import logging
+import sqlite3
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+
+from jobspy import scrape_jobs
+
+from divapply import config
+from divapply.database import get_connection, init_db, store_jobs
+
+log = logging.getLogger(__name__)
+
+
+# -- Proxy parsing -----------------------------------------------------------
+
+def parse_proxy(proxy_str: str) -> dict:
+    """Parse host:port:user:pass into components."""
+    parts = proxy_str.split(":")
+    if len(parts) == 4:
+        host, port, user, passwd = parts
+        return {
+            "host": host,
+            "port": port,
+            "user": user,
+            "pass": passwd,
+            "jobspy": f"{user}:{passwd}@{host}:{port}",
+            "playwright": {
+                "server": f"http://{host}:{port}",
+                "username": user,
+                "password": passwd,
+            },
+        }
+    elif len(parts) == 2:
+        host, port = parts
+        return {
+            "host": host,
+            "port": port,
+            "user": None,
+            "pass": None,
+            "jobspy": f"{host}:{port}",
+            "playwright": {"server": f"http://{host}:{port}"},
+        }
+    else:
+        raise ValueError(
+            f"Proxy format not recognized: {proxy_str}. "
+            f"Expected: host:port:user:pass or host:port"
+        )
+
+
+# -- Retry wrapper -----------------------------------------------------------
+
+def _scrape_with_retry(kwargs: dict, max_retries: int = 2, backoff: float = 5.0):
+    """Call scrape_jobs with retry on transient failures."""
+    for attempt in range(max_retries + 1):
+        try:
+            return scrape_jobs(**kwargs)
+        except Exception as e:
+            err = str(e).lower()
+            transient = any(k in err for k in ("timeout", "429", "proxy", "connection", "reset", "refused"))
+            if transient and attempt < max_retries:
+                wait = backoff * (attempt + 1)
+                log.warning("Retry %d/%d in %.0fs: %s", attempt + 1, max_retries, wait, e)
+                time.sleep(wait)
+            else:
+                raise
+
+
+# -- Location filtering ------------------------------------------------------
+
+def _load_location_config(search_cfg: dict) -> tuple[list[str], list[str]]:
+    """Extract accept/reject location lists from search config.
+
+    Falls back to sensible defaults if not defined in the YAML.
+    """
+    accept = search_cfg.get("location_accept", [])
+    reject = search_cfg.get("location_reject_non_remote", [])
+    return accept, reject
+
+
+def _load_title_excludes(search_cfg: dict) -> list[str]:
+    """Load title exclusion patterns from search config (case-insensitive)."""
+    return [t.lower() for t in search_cfg.get("exclude_titles", [])]
+
+
+def _title_ok(title: str | None, excludes: list[str]) -> bool:
+    """Return False if title matches any exclude pattern."""
+    if not title or not excludes:
+        return True
+    t = title.lower()
+    return not any(ex in t for ex in excludes)
+
+
+def _location_ok(location: str | None, accept: list[str], reject: list[str]) -> bool:
+    """Check if a job location passes the user's location filter.
+
+    Remote jobs are always accepted. Non-remote jobs must match an accept
+    pattern and not match a reject pattern.
+    """
+    if not location:
+        return True  # unknown location -- keep it, let scorer decide
+
+    loc = location.lower()
+
+    # Remote jobs always OK
+    if any(r in loc for r in ("remote", "anywhere", "work from home", "wfh", "distributed")):
+        return True
+
+    # Reject non-remote matches
+    for r in reject:
+        if r.lower() in loc:
+            return False
+
+    # Accept matches
+    for a in accept:
+        if a.lower() in loc:
+            return True
+
+    # No match -- reject unknown
+    return False
+
+
+# -- DB storage (JobSpy DataFrame -> SQLite) ---------------------------------
+
+def store_jobspy_results(conn: sqlite3.Connection, df, source_label: str) -> tuple[int, int]:
+    """Store JobSpy DataFrame results into the DB. Returns (new, existing)."""
+    now = datetime.now(timezone.utc).isoformat()
+    new = 0
+    existing = 0
+
+    for _, row in df.iterrows():
+        url = str(row.get("job_url", ""))
+        if not url or url == "nan":
+            continue
+
+        title = str(row.get("title", "")) if str(row.get("title", "")) != "nan" else None
+        company = str(row.get("company", "")) if str(row.get("company", "")) != "nan" else None
+        location_str = str(row.get("location", "")) if str(row.get("location", "")) != "nan" else None
+
+        # Build salary string from min/max
+        salary = None
+        min_amt = row.get("min_amount")
+        max_amt = row.get("max_amount")
+        interval = str(row.get("interval", "")) if str(row.get("interval", "")) != "nan" else ""
+        currency = str(row.get("currency", "")) if str(row.get("currency", "")) != "nan" else ""
+        if min_amt and str(min_amt) != "nan":
+            if max_amt and str(max_amt) != "nan":
+                salary = f"{currency}{int(float(min_amt)):,}-{currency}{int(float(max_amt)):,}"
+            else:
+                salary = f"{currency}{int(float(min_amt)):,}"
+            if interval:
+                salary += f"/{interval}"
+
+        description = str(row.get("description", "")) if str(row.get("description", "")) != "nan" else None
+        site_name = str(row.get("site", source_label))
+        is_remote = row.get("is_remote", False)
+
+        site_label = f"{site_name}"
+        if is_remote:
+            location_str = f"{location_str} (Remote)" if location_str else "Remote"
+
+        strategy = "jobspy"
+
+        # If JobSpy gave us a full description, promote it directly
+        full_description = None
+        detail_scraped_at = None
+        if description and len(description) > 200:
+            full_description = description
+            detail_scraped_at = now
+
+        # Extract apply URL if JobSpy provided it
+        apply_url = str(row.get("job_url_direct", "")) if str(row.get("job_url_direct", "")) != "nan" else None
+
+        try:
+            conn.execute(
+                "INSERT INTO jobs (url, title, salary, description, location, site, strategy, discovered_at, "
+                "full_description, application_url, detail_scraped_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (url, title, salary, description, location_str, site_label, strategy, now,
+                 full_description, apply_url, detail_scraped_at),
+            )
+            new += 1
+        except sqlite3.IntegrityError:
+            existing += 1
+
+    conn.commit()
+    return new, existing
+
+
+# -- Single search execution -------------------------------------------------
+
+def _run_one_search(
+    search: dict,
+    sites: list[str],
+    results_per_site: int,
+    hours_old: int,
+    proxy_config: dict | None,
+    defaults: dict,
+    max_retries: int,
+    accept_locs: list[str],
+    reject_locs: list[str],
+    glassdoor_map: dict,
+    title_excludes: list[str] | None = None,
+) -> dict:
+    """Run a single search query and store results in DB."""
+    s = search
+    label = f"\"{s['query']}\" in {s['location']} {'(remote)' if s.get('remote') else ''}"
+    if "tier" in s:
+        label += f" [tier {s['tier']}]"
+
+    # Split sites: Glassdoor needs simplified location, others use original
+    gd_location = glassdoor_map.get(s["location"], s["location"].split(",")[0])
+    has_glassdoor = "glassdoor" in sites
+    other_sites = [si for si in sites if si != "glassdoor"]
+
+    all_dfs = []
+
+    # Run non-Glassdoor sites with original location
+    if other_sites:
+        kwargs = {
+            "site_name": other_sites,
+            "search_term": s["query"],
+            "location": s["location"],
+            "results_wanted": results_per_site,
+            "hours_old": hours_old,
+            "description_format": "markdown",
+            "country_indeed": defaults.get("country_indeed", "usa"),
+            "verbose": 0,
+        }
+        if s.get("remote"):
+            kwargs["is_remote"] = True
+        if proxy_config:
+            kwargs["proxies"] = [proxy_config["jobspy"]]
+        if "linkedin" in other_sites:
+            kwargs["linkedin_fetch_description"] = True
+        try:
+            df = _scrape_with_retry(kwargs, max_retries=max_retries)
+            all_dfs.append(df)
+        except Exception as e:
+            log.error("[%s] (non-gd): %s", label, e)
+
+    # Run Glassdoor separately with simplified location
+    if has_glassdoor:
+        gd_kwargs = {
+            "site_name": ["glassdoor"],
+            "search_term": s["query"],
+            "location": gd_location,
+            "results_wanted": results_per_site,
+            "hours_old": hours_old,
+            "description_format": "markdown",
+            "verbose": 0,
+        }
+        if s.get("remote"):
+            gd_kwargs["is_remote"] = True
+        if proxy_config:
+            gd_kwargs["proxies"] = [proxy_config["jobspy"]]
+        try:
+            gd_df = _scrape_with_retry(gd_kwargs, max_retries=max_retries)
+            all_dfs.append(gd_df)
+        except Exception as e:
+            log.error("[%s] (glassdoor): %s", label, e)
+
+    if not all_dfs:
+        log.error("[%s]: all sites failed", label)
+        return {"new": 0, "existing": 0, "errors": 1, "filtered": 0, "total": 0, "label": label}
+
+    import pandas as pd
+    import warnings
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", FutureWarning)
+        df = pd.concat(all_dfs, ignore_index=True) if len(all_dfs) > 1 else all_dfs[0]
+
+    if len(df) == 0:
+        log.info("[%s] 0 results", label)
+        return {"new": 0, "existing": 0, "errors": 0, "filtered": 0, "total": 0, "label": label}
+
+    # Filter by location before storing
+    before = len(df)
+    df = df[df.apply(lambda row: _location_ok(
+        str(row.get("location", "")) if str(row.get("location", "")) != "nan" else None,
+        accept_locs, reject_locs,
+    ), axis=1)]
+    filtered_loc = before - len(df)
+
+    # Filter by title exclusion list
+    filtered_title = 0
+    if title_excludes:
+        before_title = len(df)
+        df = df[df.apply(lambda row: _title_ok(
+            str(row.get("title", "")) if str(row.get("title", "")) != "nan" else None,
+            title_excludes,
+        ), axis=1)]
+        filtered_title = before_title - len(df)
+
+    conn = get_connection()
+    new, existing = store_jobspy_results(conn, df, s["query"])
+
+    msg = f"[{label}] {before} results -> {new} new, {existing} dupes"
+    if filtered_loc:
+        msg += f", {filtered_loc} filtered (location)"
+    if filtered_title:
+        msg += f", {filtered_title} filtered (title)"
+    log.info(msg)
+
+    return {
+        "new": new,
+        "existing": existing,
+        "errors": 0,
+        "filtered": filtered_loc + filtered_title,
+        "total": before,
+        "label": label,
+    }
+
+
+# -- Single query search -----------------------------------------------------
+
+def search_jobs(
+    query: str,
+    location: str,
+    sites: list[str] | None = None,
+    remote_only: bool = False,
+    results_per_site: int = 50,
+    hours_old: int = 72,
+    proxy: str | None = None,
+    country_indeed: str = "usa",
+) -> dict:
+    """Run a single job search via JobSpy and store results in DB."""
+    if sites is None:
+        sites = ["indeed", "linkedin", "zip_recruiter"]
+
+    proxy_config = parse_proxy(proxy) if proxy else None
+
+    log.info("Search: \"%s\" in %s | sites=%s | remote=%s", query, location, sites, remote_only)
+
+    kwargs = {
+        "site_name": sites,
+        "search_term": query,
+        "location": location,
+        "results_wanted": results_per_site,
+        "hours_old": hours_old,
+        "description_format": "markdown",
+        "country_indeed": country_indeed,
+        "verbose": 2,
+    }
+
+    if remote_only:
+        kwargs["is_remote"] = True
+
+    if proxy_config:
+        kwargs["proxies"] = [proxy_config["jobspy"]]
+
+    if "linkedin" in sites:
+        kwargs["linkedin_fetch_description"] = True
+
+    try:
+        df = scrape_jobs(**kwargs)
+    except Exception as e:
+        log.error("JobSpy search failed: %s", e)
+        return {"error": str(e), "total": 0, "new": 0, "existing": 0}
+
+    total = len(df)
+    log.info("JobSpy returned %d results", total)
+
+    if total == 0:
+        return {"total": 0, "new": 0, "existing": 0}
+
+    if "site" in df.columns:
+        site_counts = df["site"].value_counts()
+        for site, count in site_counts.items():
+            log.info("  %s: %d", site, count)
+
+    conn = init_db()
+    new, existing = store_jobspy_results(conn, df, query)
+    log.info("Stored: %d new, %d already in DB", new, existing)
+
+    db_total = conn.execute("SELECT COUNT(*) FROM jobs").fetchone()[0]
+    pending = conn.execute("SELECT COUNT(*) FROM jobs WHERE detail_scraped_at IS NULL").fetchone()[0]
+    log.info("DB total: %d jobs, %d pending detail scrape", db_total, pending)
+
+    return {"total": total, "new": new, "existing": existing}
+
+
+# -- Full crawl (all queries x all locations) --------------------------------
+
+def _full_crawl(
+    search_cfg: dict,
+    tiers: list[int] | None = None,
+    locations: list[str] | None = None,
+    sites: list[str] | None = None,
+    results_per_site: int = 100,
+    hours_old: int = 72,
+    proxy: str | None = None,
+    max_retries: int = 2,
+    workers: int = 4,
+) -> dict:
+    """Run all search queries from search config across all locations.
+
+    Workers > 1 runs multiple searches in parallel (network I/O bound).
+    Each worker uses a randomised delay so job boards see staggered requests.
+    """
+    if sites is None:
+        sites = ["indeed", "linkedin", "zip_recruiter"]
+
+    # Build search combinations from config
+    queries = search_cfg.get("queries", [])
+    locs = search_cfg.get("locations", [])
+    defaults = search_cfg.get("defaults", {})
+    glassdoor_map = search_cfg.get("glassdoor_location_map", {})
+    accept_locs, reject_locs = _load_location_config(search_cfg)
+    title_excludes = _load_title_excludes(search_cfg)
+
+    if tiers:
+        queries = [q for q in queries if q.get("tier") in tiers]
+    if locations:
+        locs = [loc for loc in locs if loc.get("label") in locations]
+
+    searches = []
+    for q in queries:
+        for loc in locs:
+            searches.append({
+                "query": q["query"],
+                "location": loc["location"],
+                "remote": loc.get("remote", False),
+                "tier": q.get("tier", 0),
+            })
+
+    # Support per-worker proxy rotation: proxy can be a single string or
+    # a comma-separated list ("host:port:user:pass,host2:port2:user2:pass2").
+    proxy_list: list[dict | None] = [None]
+    if proxy:
+        raw_proxies = [p.strip() for p in proxy.split(",") if p.strip()]
+        proxy_list = [parse_proxy(p) for p in raw_proxies]
+
+    log.info("Full crawl: %d search combinations | workers=%d | proxies=%d",
+             len(searches), workers, len([p for p in proxy_list if p]))
+    log.info("Sites: %s | Results/site: %d | Hours old: %d",
+             ", ".join(sites), results_per_site, hours_old)
+
+    # Ensure DB schema is ready
+    init_db()
+
+    _lock = threading.Lock()
+    total_new = 0
+    total_existing = 0
+    total_errors = 0
+    completed = 0
+
+    def _run_search(idx_search: tuple[int, dict]) -> dict:
+        import random
+        idx, s = idx_search
+        # Round-robin proxy assignment per search
+        proxy_cfg = proxy_list[idx % len(proxy_list)]
+        # Stagger start times so workers don't all hit the same site at once
+        time.sleep(random.uniform(0, 1.5) * (idx % workers))
+        return _run_one_search(
+            s, sites, results_per_site, hours_old,
+            proxy_cfg, defaults, max_retries,
+            accept_locs, reject_locs, glassdoor_map,
+            title_excludes,
+        )
+
+    if workers > 1:
+        with ThreadPoolExecutor(max_workers=workers,
+                                thread_name_prefix="jobspy") as pool:
+            futures = {
+                pool.submit(_run_search, (i, s)): s
+                for i, s in enumerate(searches)
+            }
+            for future in as_completed(futures):
+                result = future.result()
+                with _lock:
+                    total_new += result["new"]
+                    total_existing += result["existing"]
+                    total_errors += result["errors"]
+                    completed += 1
+                    if completed % 5 == 0 or completed == len(searches):
+                        log.info("Progress: %d/%d queries done (%d new, %d dupes, %d errors)",
+                                 completed, len(searches), total_new, total_existing, total_errors)
+    else:
+        for i, s in enumerate(searches):
+            result = _run_search((i, s))
+            total_new += result["new"]
+            total_existing += result["existing"]
+            total_errors += result["errors"]
+            completed += 1
+            if completed % 5 == 0 or completed == len(searches):
+                log.info("Progress: %d/%d queries done (%d new, %d dupes, %d errors)",
+                         completed, len(searches), total_new, total_existing, total_errors)
+
+    # Final stats
+    conn = get_connection()
+    db_total = conn.execute("SELECT COUNT(*) FROM jobs").fetchone()[0]
+
+    log.info("Full crawl complete: %d new | %d dupes | %d errors | %d total in DB",
+             total_new, total_existing, total_errors, db_total)
+
+    return {
+        "new": total_new,
+        "existing": total_existing,
+        "errors": total_errors,
+        "db_total": db_total,
+        "queries": len(searches),
+    }
+
+
+# -- Public entry point ------------------------------------------------------
+
+def run_discovery(cfg: dict | None = None, workers: int = 4) -> dict:
+    """Main entry point for JobSpy-based job discovery.
+
+    Loads search queries and locations from the user's search config YAML,
+    then runs a full crawl across all configured job boards.
+
+    Args:
+        cfg: Override the search configuration dict. If None, loads from
+             the user's searches.yaml file.
+
+    Returns:
+        Dict with stats: new, existing, errors, db_total, queries.
+    """
+    if cfg is None:
+        cfg = config.load_search_config()
+
+    if not cfg:
+        log.warning("No search configuration found. Run `divapply init` to create one.")
+        return {"new": 0, "existing": 0, "errors": 0, "db_total": 0, "queries": 0}
+
+    proxy = cfg.get("proxy")
+    sites = cfg.get("sites")
+    results_per_site = cfg.get("defaults", {}).get("results_per_site", 100)
+    hours_old = cfg.get("defaults", {}).get("hours_old", 72)
+    tiers = cfg.get("tiers")
+    locations = cfg.get("location_labels")
+
+    crawl_workers = cfg.get("defaults", {}).get("workers", workers)
+
+    return _full_crawl(
+        search_cfg=cfg,
+        tiers=tiers,
+        locations=locations,
+        sites=sites,
+        results_per_site=results_per_site,
+        hours_old=hours_old,
+        proxy=proxy,
+        workers=crawl_workers,
+    )
+