stage-cli 1.0.0__py3-none-any.whl

stage/normalize/location.py

@@ -0,0 +1,271 @@
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+
+from stage.domain import LocationBucket, RemoteScope
+from stage.lexicon import LocationLexicon, fold, location_lexicon
+
+_SEGMENT_SPLIT = re.compile(r"[;/•|\n]+")
+_FIELD_SPLIT = re.compile(r"[,\-]+")
+_SUBDIVISION_CODE = re.compile(r"[A-Z]{1,3}")
+_COUNTRY_PREFIXES = frozenset({"can", "us", "usa"})
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedLocation:
+    bucket: LocationBucket = LocationBucket.UNKNOWN
+    remote_scope: RemoteScope | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _Segment:
+    montreal: bool
+    canada: bool
+    usa: bool
+    international: bool
+    remote: bool
+
+
+class _PhraseIndex:
+    __slots__ = ("_by_first_token",)
+
+    def __init__(self, categories: dict[str, frozenset[str]]) -> None:
+        index: dict[str, list[tuple[str, str]]] = {}
+        for category, phrases in categories.items():
+            for phrase in phrases:
+                index.setdefault(phrase.split(" ", 1)[0], []).append((category, phrase))
+        self._by_first_token = {token: tuple(entries) for token, entries in index.items()}
+
+    def hits(self, folded: str) -> set[tuple[str, str]]:
+        padded = f" {folded} "
+        found: set[tuple[str, str]] = set()
+        for token in folded.split():
+            for entry in self._by_first_token.get(token, ()):
+                if f" {entry[1]} " in padded:
+                    found.add(entry)
+        return found
+
+
+@lru_cache(maxsize=1)
+def _index() -> _PhraseIndex:
+    lexicon = location_lexicon()
+    return _PhraseIndex(
+        {
+            "montreal": lexicon.montreal,
+            "montreal_ambiguous": lexicon.montreal_ambiguous,
+            "canada_cities": lexicon.canada_cities,
+            "canada_ambiguous": lexicon.canada_ambiguous,
+            "canada_regions": lexicon.canada_regions,
+            "canada_country": lexicon.canada_country,
+            "canada_overrides": lexicon.canada_overrides,
+            "usa_ambiguous": lexicon.usa_ambiguous,
+            "usa_cities": lexicon.usa_cities,
+            "usa_regions": lexicon.usa_regions,
+            "usa_country": lexicon.usa_country,
+            "international": lexicon.international,
+            "international_cities": lexicon.international_cities,
+            "remote": lexicon.remote,
+        }
+    )
+
+
+def _foreign_subdivision_before(fields: list[str], index: int, known: frozenset[str]) -> bool:
+    for previous in reversed(fields[:index]):
+        stripped = previous.strip()
+        if not stripped:
+            continue
+        if not _SUBDIVISION_CODE.fullmatch(stripped):
+            return False
+        folded = fold(stripped)
+        return folded not in known and folded not in _COUNTRY_PREFIXES
+    return False
+
+
+def _last_alphabetic_field(fields: list[str]) -> int:
+    for index in reversed(range(len(fields))):
+        if any(character.isalpha() for character in fields[index]):
+            return index
+    return -1
+
+
+def _reads_as_country(fields: list[str], index: int, token: str, known: frozenset[str]) -> bool:
+    if index != _last_alphabetic_field(fields):
+        return False
+    if fold(fields[index].strip()) != token:
+        return False
+    return _foreign_subdivision_before(fields, index, known)
+
+
+def _code_hits(segment: str, lexicon: LocationLexicon) -> tuple[bool, bool]:
+    canada = usa = False
+    fields = _FIELD_SPLIT.split(segment)
+    known = lexicon.canada_codes | lexicon.usa_codes
+    for index, field in enumerate(fields):
+        for token in fold(field).split():
+            if token not in known:
+                continue
+            if not re.search(rf"\b{token.upper()}\b", segment):
+                continue
+            if _reads_as_country(fields, index, token, known):
+                continue
+            if token in lexicon.canada_codes:
+                canada = True
+            else:
+                usa = True
+    return canada, usa
+
+
+_FOREIGN_ONLY_CODES = frozenset({"de", "es", "ie", "in", "it", "fr", "uk", "nl", "pl", "br", "cn"})
+
+
+def _trails_a_foreign_code(segment: str) -> bool:
+    lexicon = location_lexicon()
+    fields = [field.strip() for field in _FIELD_SPLIT.split(segment) if field.strip()]
+    if len(fields) < 2:
+        return False
+    known = lexicon.canada_codes | lexicon.usa_codes
+    for field in fields[1:]:
+        folded = fold(field)
+        if folded in _FOREIGN_ONLY_CODES:
+            return True
+        if folded and len(folded) <= 3 and folded not in known and not folded.isdigit():
+            return True
+    return False
+
+
+def _resolve_segment(segment: str, lexicon: LocationLexicon) -> _Segment:
+    folded = fold(segment)
+    if not folded:
+        return _Segment(False, False, False, False, False)
+    hits = _index().hits(folded)
+    found = {category for category, _ in hits}
+    international_phrases = {
+        phrase for category, phrase in hits if category in {"international", "international_cities"}
+    }
+    domestic_phrases = {
+        phrase
+        for category, phrase in hits
+        if category in {"canada_cities", "canada_regions", "usa_cities", "usa_regions"}
+    }
+    international = any(
+        not any(
+            len(domestic) > len(phrase) and f" {phrase} " in f" {domestic} "
+            for domestic in domestic_phrases
+        )
+        for phrase in international_phrases
+    )
+    code_canada, code_usa = _code_hits(segment, lexicon)
+
+    override_phrases = {phrase for category, phrase in hits if category == "canada_overrides"}
+    named_canada = {
+        phrase
+        for category, phrase in hits
+        if category in {"canada_cities", "canada_ambiguous", "montreal", "montreal_ambiguous"}
+    }
+    overridden = bool(override_phrases) and all(
+        any(f" {phrase} " in f" {override} " for override in override_phrases)
+        for phrase in named_canada
+    )
+    ambiguous_here = bool(found & {"canada_ambiguous", "montreal_ambiguous", "usa_ambiguous"})
+    coded = (
+        (code_canada or code_usa)
+        and ambiguous_here
+        and not _trails_a_foreign_code(segment)
+        and not found & {"international"}
+    )
+    canada_context = (
+        "canada_country" in found
+        or "canada_regions" in found
+        or (code_canada and (not international or coded))
+    ) and not overridden
+
+    montreal = ("montreal" in found and not overridden) or (
+        "montreal_ambiguous" in found and canada_context
+    )
+    canada = (
+        montreal
+        or canada_context
+        or ("canada_cities" in found and not overridden and not international)
+        or ("canada_ambiguous" in found and canada_context)
+    )
+    usa = "usa_country" in found or (
+        ("usa_regions" in found or "usa_cities" in found or code_usa)
+        and (not international or (code_usa and coded))
+    )
+    return _Segment(
+        montreal=montreal,
+        canada=canada,
+        usa=usa,
+        international=international and not (coded and (canada or usa)),
+        remote="remote" in found,
+    )
+
+
+def _scope(segments: list[_Segment]) -> RemoteScope | None:
+    scopes = {
+        RemoteScope.CANADA
+        if segment.canada
+        else RemoteScope.US
+        if segment.usa
+        else RemoteScope.UNSPECIFIED
+        for segment in segments
+        if segment.remote
+    }
+    if not scopes:
+        return None
+    for candidate in (RemoteScope.CANADA, RemoteScope.UNSPECIFIED, RemoteScope.US):
+        if candidate in scopes:
+            return candidate
+    return RemoteScope.UNSPECIFIED
+
+
+_MULTI_SITE_SUFFIX = re.compile(r"\s*\+\s*\d+\s*$")
+
+
+def display_location(raw: str) -> str:
+    if not raw or not raw.strip():
+        return ""
+
+    aliases = location_lexicon().city_aliases
+    first = _MULTI_SITE_SUFFIX.sub("", _SEGMENT_SPLIT.split(raw)[0].strip())
+    if not first:
+        return raw.strip()
+
+    others = len([part for part in _SEGMENT_SPLIT.split(raw)[1:] if part.strip()])
+    more = f" +{others}" if others else ""
+
+    fields = [field.strip() for field in first.split(",") if field.strip()]
+    if not fields:
+        return raw.strip()
+
+    canonical = aliases.get(fold(fields[0]))
+    if canonical is None:
+        return f"{first}{more}"
+    return ", ".join([canonical, *fields[1:]]) + more
+
+
+def resolve_location(raw: str) -> ResolvedLocation:
+    if not raw or not raw.strip():
+        return ResolvedLocation()
+
+    lexicon = location_lexicon()
+    segments = [
+        _resolve_segment(part, lexicon) for part in _SEGMENT_SPLIT.split(raw) if part.strip()
+    ]
+    if not segments:
+        return ResolvedLocation()
+
+    scope = _scope(segments)
+
+    if any(segment.montreal for segment in segments):
+        bucket = LocationBucket.MONTREAL
+    elif any(segment.canada for segment in segments):
+        bucket = LocationBucket.CANADA
+    elif any(segment.usa for segment in segments):
+        bucket = LocationBucket.USA
+    elif any(segment.international for segment in segments):
+        bucket = LocationBucket.INTERNATIONAL
+    else:
+        bucket = LocationBucket.UNKNOWN
+
+    return ResolvedLocation(bucket=bucket, remote_scope=scope)

stage/normalize/terms.py

@@ -0,0 +1,153 @@
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from stage.domain import UNKNOWN_TERM
+from stage.lexicon import fold, term_lexicon
+
+_YEAR = re.compile(r"^(20[2-3]\d)$")
+_SHORT_YEAR = re.compile(r"^(\d{2})$")
+_PROXIMITY = 2
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedTerm:
+    term: str = UNKNOWN_TERM
+    season: str = ""
+    year: int | None = None
+    evidence: tuple[str, ...] = ()
+    conflict: bool = False
+
+
+def _year_from(token: str, pivot_year: int | None) -> int | None:
+    full = _YEAR.match(token)
+    if full:
+        return int(full.group(1))
+    short = _SHORT_YEAR.match(token)
+    if short and pivot_year is not None:
+        candidate = pivot_year - pivot_year % 100 + int(short.group(1))
+        if abs(candidate - pivot_year) <= 5:
+            return candidate
+    return None
+
+
+def _blocked(tokens: Sequence[str], index: int, blocked: frozenset[str]) -> bool:
+    if index + 1 < len(tokens) and f"{tokens[index]} {tokens[index + 1]}" in blocked:
+        return True
+    return index > 0 and f"{tokens[index - 1]} {tokens[index]}" in blocked
+
+
+def scan(text: str, pivot_year: int | None = None) -> list[tuple[str, int | None, str]]:
+    lexicon = term_lexicon()
+    tokens = fold(text).split()
+    if not tokens:
+        return []
+
+    surface: dict[str, str] = {}
+    for name, phrases in lexicon.seasons.items():
+        for phrase in phrases:
+            surface[phrase] = name
+
+    found: list[tuple[str, int | None, str]] = []
+    for index, token in enumerate(tokens):
+        matched = surface.get(token)
+        if matched is None or _blocked(tokens, index, lexicon.blocked_bigrams):
+            continue
+        season = matched
+        year: int | None = None
+        phrase = token
+        for offset in range(1, _PROXIMITY + 2):
+            ahead = index + offset
+            if ahead >= len(tokens):
+                break
+            candidate = _year_from(tokens[ahead], pivot_year)
+            if candidate is not None:
+                year, phrase = candidate, " ".join(tokens[index : ahead + 1])
+                break
+            if tokens[ahead] not in lexicon.fillers:
+                break
+        if year is None:
+            behind = index - 1
+            if behind >= 0:
+                candidate = _year_from(tokens[behind], pivot_year)
+                if candidate is not None:
+                    year, phrase = candidate, f"{tokens[behind]} {token}"
+        found.append((season, year, phrase))
+    return found
+
+
+def _terms_in(text: str, pivot_year: int | None) -> tuple[set[str], set[str], list[str]]:
+    terms: set[str] = set()
+    seasons: set[str] = set()
+    evidence: list[str] = []
+    for season, year, phrase in scan(text, pivot_year):
+        seasons.add(season)
+        evidence.append(phrase)
+        if year is not None:
+            terms.add(f"{season}-{year}")
+    return terms, seasons, evidence
+
+
+def _structured(
+    values: Sequence[str], season: str, pivot_year: int | None
+) -> tuple[set[str], set[str], list[str]]:
+    terms: set[str] = set()
+    seasons: set[str] = set()
+    evidence: list[str] = []
+    for value in [*values, season]:
+        if not value or value.strip().lower() in {"n/a", "na", "none", "null", "unknown"}:
+            continue
+        found, found_seasons, phrases = _terms_in(value, pivot_year)
+        terms |= found
+        seasons |= found_seasons
+        evidence.extend(phrases)
+    return terms, seasons, evidence
+
+
+def resolve_term(
+    *,
+    title: str = "",
+    description: str = "",
+    structured_terms: Sequence[str] = (),
+    structured_season: str = "",
+    pivot_year: int | None = None,
+) -> ResolvedTerm:
+    authored = (
+        _structured(structured_terms, structured_season, pivot_year),
+        _terms_in(title, pivot_year),
+    )
+    evidence: list[str] = []
+    seasons: set[str] = set()
+    speaking: list[set[str]] = []
+    for terms, found_seasons, phrases in authored:
+        evidence.extend(phrases)
+        seasons |= found_seasons
+        if terms:
+            speaking.append(terms)
+
+    if not speaking:
+        body_terms, body_seasons, body_phrases = _terms_in(description, pivot_year)
+        evidence.extend(body_phrases)
+        seasons |= body_seasons
+        if body_terms:
+            speaking.append(body_terms)
+
+    unique = tuple(dict.fromkeys(evidence))
+    season = next(iter(sorted(seasons))) if len(seasons) == 1 else ""
+
+    if not speaking:
+        return ResolvedTerm(season=season, evidence=unique)
+
+    conflict = len({frozenset(terms) for terms in speaking}) > 1
+    chosen = speaking[0]
+    if conflict or len(chosen) != 1:
+        return ResolvedTerm(season=season, evidence=unique, conflict=conflict)
+
+    term = next(iter(chosen))
+    resolved_season, _, year = term.partition("-")
+    return ResolvedTerm(
+        term=term,
+        season=resolved_season,
+        year=int(year),
+        evidence=unique,
+    )

stage/normalize/urls.py

@@ -0,0 +1,122 @@
+import re
+from urllib.parse import SplitResult, parse_qsl, urlsplit, urlunsplit
+
+TRACKER_DOMAINS: frozenset[str] = frozenset(
+    {
+        "simplify.jobs",
+        "click.appcast.io",
+        "trk.simplify.jobs",
+        "jobright.ai",
+        "app.otta.com",
+        "get.hiring.cafe",
+        "grnh.se",
+        "boards.greenhouse.io.simplify.jobs",
+        "l.workwithus.io",
+        "track.rippling-ats.com",
+    }
+)
+
+LOCALE_LANGUAGES: frozenset[str] = frozenset(
+    {
+        "ar",
+        "cs",
+        "da",
+        "de",
+        "el",
+        "en",
+        "es",
+        "fi",
+        "fr",
+        "he",
+        "hu",
+        "ja",
+        "ko",
+        "nl",
+        "pl",
+        "pt",
+        "ro",
+        "ru",
+        "sv",
+        "th",
+        "tr",
+        "vi",
+        "zh",
+    }
+)
+
+IDENTITY_PARAMS: frozenset[str] = frozenset(
+    {
+        "gh_jid",
+        "id",
+        "jid",
+        "job",
+        "jobid",
+        "job_id",
+        "jobnumber",
+        "jobreqid",
+        "opportunityid",
+        "pid",
+        "positionid",
+        "postingid",
+        "req",
+        "reqid",
+        "req_id",
+        "requisitionid",
+        "rid",
+        "vacancyid",
+    }
+)
+
+_LOCALE_REGION = re.compile(r"^[a-z]{2}[-_][a-zA-Z]{2}$")
+_IDENTITY_VALUE = re.compile(r"^[A-Za-z0-9._~-]{1,128}$")
+_IDENTITY_KEYS: frozenset[str] = frozenset(name.replace("_", "") for name in IDENTITY_PARAMS)
+
+
+def _is_locale_segment(segment: str) -> bool:
+    return bool(_LOCALE_REGION.match(segment)) or segment.lower() in LOCALE_LANGUAGES
+
+
+def _split(url: str) -> SplitResult | None:
+    try:
+        return urlsplit(url)
+    except ValueError:
+        return None
+
+
+def _host(url: str) -> str:
+    parts = _split(url)
+    return "" if parts is None else (parts.hostname or "").lower()
+
+
+def is_tracker_url(url: str) -> bool:
+    host = _host(url)
+    if not host:
+        return False
+    return any(host == domain or host.endswith(f".{domain}") for domain in TRACKER_DOMAINS)
+
+
+def _identity_query(query: str) -> str:
+    if not query:
+        return ""
+    kept: list[str] = []
+    for name, value in parse_qsl(query, keep_blank_values=False):
+        if name.lower().replace("-", "_").replace("_", "") not in _IDENTITY_KEYS:
+            continue
+        if _IDENTITY_VALUE.match(value):
+            kept.append(f"{name.lower()}={value}")
+    return "&".join(sorted(kept))
+
+
+def canonical_apply_url(raw: str) -> str:
+    if not raw or not raw.strip():
+        return ""
+    parts = _split(raw.strip())
+    if parts is None or parts.scheme not in ("http", "https"):
+        return ""
+    host = (parts.hostname or "").lower()
+    if not host or is_tracker_url(raw):
+        return ""
+    segments = [segment for segment in parts.path.split("/") if segment]
+    kept = [segment for segment in segments if not _is_locale_segment(segment)]
+    path = "/" + "/".join(kept)
+    return urlunsplit(("https", host, path.rstrip("/") or "/", _identity_query(parts.query), ""))

stage/paths.py

@@ -0,0 +1,86 @@
+import getpass
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+from platformdirs import PlatformDirs
+
+APP_NAME = "stage"
+_DIRS = PlatformDirs(appname=APP_NAME, appauthor=False, roaming=False)
+
+
+def data_dir() -> Path:
+    path = Path(_DIRS.user_data_dir)
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def config_dir() -> Path:
+    path = Path(_DIRS.user_config_dir)
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def database_path() -> Path:
+    override = os.environ.get("STAGE_DB")
+    if override:
+        return Path(override).expanduser().resolve()
+    return data_dir() / "stage.db"
+
+
+def registry_path() -> Path:
+    override = os.environ.get("STAGE_REGISTRY")
+    if override:
+        return Path(override).expanduser().resolve()
+    data = Path(__file__).resolve().parent / "data"
+    for packaged in (data / "companies", data / "companies.yaml"):
+        if packaged.exists():
+            return packaged
+    return config_dir() / "companies.yaml"
+
+
+def lexicon_dir() -> Path:
+    override = os.environ.get("STAGE_LEXICON")
+    if override:
+        return Path(override).expanduser().resolve()
+    return Path(__file__).resolve().parent / "data" / "lexicon"
+
+
+def font_path() -> Path:
+    packaged = Path(__file__).resolve().parent / "data" / "fonts" / "DejaVuSans.ttf"
+    if not packaged.exists():
+        raise FileNotFoundError(
+            f"the embedded PDF font is missing from {packaged.parent} — "
+            "export --format csv needs no font"
+        )
+    return packaged
+
+
+def capture_dir() -> Path:
+    override = os.environ.get("STAGE_CAPTURE_DIR")
+    root = Path(override).expanduser() if override else data_dir() / "captured"
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def restrict_permissions(path: Path) -> None:
+    if os.name == "nt":
+        _restrict_windows_permissions(path)
+        return
+    path.chmod(stat.S_IRUSR | stat.S_IWUSR)
+
+
+def _restrict_windows_permissions(path: Path) -> None:
+    root = os.environ.get("SYSTEMROOT", r"C:\Windows").rstrip("\\/")
+    executable = f"{root}\\System32\\icacls.exe"
+    principal = f"{os.environ.get('USERDOMAIN', '.')}\\{getpass.getuser()}"
+    result = subprocess.run(
+        (executable, str(path), "/inheritance:r", "/grant:r", f"{principal}:(F)"),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout).strip()
+        raise PermissionError(f"could not restrict permissions on {path}: {detail}")