stage-cli 1.0.0__py3-none-any.whl

stage/storage/migrations/0001_initial.sql

@@ -0,0 +1,239 @@
+CREATE TABLE jobs (
+    id                  TEXT PRIMARY KEY,
+    source              TEXT NOT NULL,
+    company             TEXT NOT NULL,
+    company_fold        TEXT NOT NULL DEFAULT '',
+    title_raw           TEXT NOT NULL,
+    title_normalized    TEXT NOT NULL,
+    title_canonical     TEXT NOT NULL DEFAULT '',
+    apply_url_raw       TEXT NOT NULL,
+    apply_url_canonical TEXT NOT NULL DEFAULT '',
+    description         TEXT NOT NULL DEFAULT '',
+    location_raw        TEXT NOT NULL DEFAULT '',
+    location            TEXT NOT NULL DEFAULT 'unknown',
+    remote_scope        TEXT,
+    language            TEXT NOT NULL DEFAULT 'unknown',
+    term                TEXT NOT NULL DEFAULT 'unknown',
+    role                TEXT NOT NULL DEFAULT 'unknown',
+    degree_requirement  TEXT NOT NULL DEFAULT 'unknown',
+    work_auth_flag      INTEGER NOT NULL DEFAULT 0,
+    compensation        TEXT,
+    employment_type     TEXT NOT NULL DEFAULT '',
+    source_category     TEXT NOT NULL DEFAULT '',
+    status              TEXT NOT NULL DEFAULT 'open',
+    duplicate_of        TEXT,
+    first_seen          TEXT NOT NULL,
+    last_seen           TEXT NOT NULL,
+    source_posted_at    TEXT
+);
+
+CREATE INDEX idx_jobs_first_seen ON jobs (first_seen DESC);
+CREATE INDEX idx_jobs_status_first_seen ON jobs (status, first_seen DESC);
+CREATE INDEX idx_jobs_source_company ON jobs (source, company);
+CREATE INDEX idx_jobs_duplicate_of ON jobs (duplicate_of);
+CREATE INDEX idx_jobs_degree ON jobs (degree_requirement);
+CREATE INDEX idx_jobs_company_canonical ON jobs (company) WHERE duplicate_of IS NULL;
+CREATE INDEX idx_jobs_company_fold ON jobs (company_fold);
+CREATE INDEX idx_jobs_apply_url_canonical ON jobs (apply_url_canonical);
+
+CREATE VIRTUAL TABLE jobs_fts USING fts5 (
+    company,
+    title_raw,
+    title_normalized,
+    title_canonical,
+    location_raw,
+    description,
+    content = 'jobs',
+    content_rowid = 'rowid',
+    tokenize = 'unicode61 remove_diacritics 2'
+);
+
+CREATE TRIGGER jobs_fts_after_insert AFTER INSERT ON jobs BEGIN
+    INSERT INTO jobs_fts (
+        rowid, company, title_raw, title_normalized, title_canonical,
+        location_raw, description
+    )
+    VALUES (
+        new.rowid, new.company, new.title_raw, new.title_normalized,
+        new.title_canonical, new.location_raw, new.description
+    );
+END;
+
+CREATE TRIGGER jobs_fts_after_delete AFTER DELETE ON jobs BEGIN
+    INSERT INTO jobs_fts (
+        jobs_fts, rowid, company, title_raw, title_normalized, title_canonical,
+        location_raw, description
+    )
+    VALUES (
+        'delete', old.rowid, old.company, old.title_raw, old.title_normalized,
+        old.title_canonical, old.location_raw, old.description
+    );
+END;
+
+CREATE TRIGGER jobs_fts_after_update AFTER UPDATE ON jobs
+WHEN new.company IS NOT old.company
+    OR new.title_raw IS NOT old.title_raw
+    OR new.title_normalized IS NOT old.title_normalized
+    OR new.title_canonical IS NOT old.title_canonical
+    OR new.location_raw IS NOT old.location_raw
+    OR new.description IS NOT old.description
+BEGIN
+    INSERT INTO jobs_fts (
+        jobs_fts, rowid, company, title_raw, title_normalized, title_canonical,
+        location_raw, description
+    )
+    VALUES (
+        'delete', old.rowid, old.company, old.title_raw, old.title_normalized,
+        old.title_canonical, old.location_raw, old.description
+    );
+    INSERT INTO jobs_fts (
+        rowid, company, title_raw, title_normalized, title_canonical,
+        location_raw, description
+    )
+    VALUES (
+        new.rowid, new.company, new.title_raw, new.title_normalized,
+        new.title_canonical, new.location_raw, new.description
+    );
+END;
+
+CREATE TABLE quarantine (
+    id              TEXT PRIMARY KEY,
+    source          TEXT NOT NULL,
+    company         TEXT NOT NULL,
+    title_raw       TEXT NOT NULL,
+    apply_url_raw   TEXT NOT NULL DEFAULT '',
+    location_raw    TEXT NOT NULL DEFAULT '',
+    location        TEXT NOT NULL DEFAULT 'unknown',
+    remote_scope    TEXT,
+    reason          TEXT NOT NULL,
+    matched_phrase  TEXT NOT NULL DEFAULT '',
+    first_seen      TEXT NOT NULL,
+    last_seen       TEXT NOT NULL
+);
+
+CREATE INDEX idx_quarantine_first_seen ON quarantine (first_seen DESC);
+CREATE INDEX idx_quarantine_reason_first_seen ON quarantine (reason, first_seen DESC);
+CREATE INDEX idx_quarantine_company ON quarantine (company, last_seen DESC);
+CREATE INDEX idx_quarantine_source_company ON quarantine (source, company);
+
+CREATE TABLE tombstones (
+    id         TEXT PRIMARY KEY,
+    source     TEXT NOT NULL,
+    first_seen TEXT NOT NULL,
+    purged_at  TEXT NOT NULL
+);
+
+CREATE INDEX idx_tombstones_purged_at ON tombstones (purged_at DESC);
+
+CREATE TABLE http_cache (
+    url           TEXT PRIMARY KEY,
+    source        TEXT NOT NULL,
+    etag          TEXT,
+    last_modified TEXT,
+    fetched_at    TEXT NOT NULL
+);
+
+CREATE INDEX idx_http_cache_source ON http_cache (source);
+
+CREATE TABLE rate_state (
+    bucket                TEXT PRIMARY KEY,
+    blocked_until         TEXT,
+    min_interval_override REAL,
+    consecutive_failures  INTEGER NOT NULL DEFAULT 0,
+    last_failure_at       TEXT,
+    reason                TEXT NOT NULL DEFAULT '',
+    rotation_cursor       TEXT NOT NULL DEFAULT '',
+    updated_at            TEXT NOT NULL
+);
+
+CREATE TABLE source_visits (
+    source               TEXT NOT NULL,
+    board                TEXT NOT NULL,
+    label                TEXT NOT NULL DEFAULT '',
+    last_attempt_at      TEXT NOT NULL,
+    last_success_at      TEXT,
+    consecutive_failures INTEGER NOT NULL DEFAULT 0,
+    last_error           TEXT NOT NULL DEFAULT '',
+    PRIMARY KEY (source, board)
+);
+
+CREATE INDEX idx_source_visits_success ON source_visits (source, last_success_at);
+
+CREATE TABLE coverage_classifications (
+    company      TEXT NOT NULL CHECK (length(trim(company)) > 0),
+    company_fold TEXT PRIMARY KEY,
+    disposition  TEXT NOT NULL CHECK (disposition IN (
+        'feed-only', 'unavailable', 'custom-json-candidate', 'adapter-candidate', 'deferred'
+    )),
+    note         TEXT NOT NULL CHECK (length(trim(note)) > 0),
+    checked_on   TEXT NOT NULL,
+    url          TEXT
+);
+
+CREATE TABLE workday_facets (
+    tenant       TEXT NOT NULL,
+    site         TEXT NOT NULL,
+    parameter    TEXT NOT NULL,
+    facet_id     TEXT NOT NULL,
+    descriptor   TEXT NOT NULL DEFAULT '',
+    resolved_at  TEXT NOT NULL,
+    PRIMARY KEY (tenant, site)
+);
+
+CREATE TABLE workday_crawls (
+    board           TEXT PRIMARY KEY,
+    next_offset     INTEGER NOT NULL CHECK (next_offset >= 0),
+    total           INTEGER,
+    facet_parameter TEXT NOT NULL DEFAULT '',
+    facet_ids       TEXT NOT NULL DEFAULT ''
+);
+
+CREATE TABLE workday_crawl_seen (
+    board TEXT NOT NULL REFERENCES workday_crawls (board) ON DELETE CASCADE,
+    id    TEXT NOT NULL,
+    PRIMARY KEY (board, id)
+);
+
+CREATE INDEX idx_workday_crawl_seen_id ON workday_crawl_seen (id);
+
+CREATE TABLE detail_fetches (
+    id         TEXT PRIMARY KEY,
+    source     TEXT NOT NULL,
+    fetched_at TEXT NOT NULL,
+    resolved   INTEGER NOT NULL DEFAULT 0,
+    attempts   INTEGER NOT NULL DEFAULT 1,
+    failed     INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX idx_detail_fetches_source ON detail_fetches (source, resolved, failed);
+
+CREATE TABLE sync_runs (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    started_at  TEXT NOT NULL,
+    finished_at TEXT NOT NULL,
+    outcome     TEXT NOT NULL
+);
+
+CREATE INDEX idx_sync_runs_started_at ON sync_runs (started_at DESC);
+
+CREATE TABLE sync_run_sources (
+    run_id         INTEGER NOT NULL REFERENCES sync_runs (id) ON DELETE CASCADE,
+    source         TEXT NOT NULL,
+    fetched        INTEGER NOT NULL DEFAULT 0,
+    added          INTEGER NOT NULL DEFAULT 0,
+    updated        INTEGER NOT NULL DEFAULT 0,
+    closed         INTEGER NOT NULL DEFAULT 0,
+    errors         INTEGER NOT NULL DEFAULT 0,
+    requests       INTEGER NOT NULL DEFAULT 0,
+    not_modified   INTEGER NOT NULL DEFAULT 0,
+    retries        INTEGER NOT NULL DEFAULT 0,
+    tightenings    INTEGER NOT NULL DEFAULT 0,
+    quarantined    INTEGER NOT NULL DEFAULT 0,
+    deferred       INTEGER NOT NULL DEFAULT 0,
+    blocked        INTEGER NOT NULL DEFAULT 0,
+    stored         INTEGER NOT NULL DEFAULT -1,
+    latency_p50_ms REAL NOT NULL DEFAULT 0,
+    latency_p95_ms REAL NOT NULL DEFAULT 0,
+    elapsed_ms     REAL NOT NULL DEFAULT 0,
+    PRIMARY KEY (run_id, source)
+);

stage/storage/migrations/__init__.py

@@ -0,0 +1,135 @@
+import re
+import sqlite3
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+_FILENAME = re.compile(r"^(\d{4})_[a-z0-9_]+\.sql$")
+_MIGRATIONS_DIR = Path(__file__).resolve().parent
+
+
+class SchemaVersionError(Exception):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class Migration:
+    version: int
+    name: str
+    path: Path
+
+    def read(self) -> str:
+        return self.path.read_text(encoding="utf-8")
+
+
+def discover() -> tuple[Migration, ...]:
+    found: list[Migration] = []
+    for path in sorted(_MIGRATIONS_DIR.glob("*.sql")):
+        match = _FILENAME.match(path.name)
+        if match is None:
+            raise SchemaVersionError(f"migration {path.name!r} does not follow NNNN_name.sql")
+        found.append(Migration(version=int(match.group(1)), name=path.stem, path=path))
+    versions = [migration.version for migration in found]
+    if len(set(versions)) != len(versions):
+        raise SchemaVersionError("duplicate migration version numbers")
+    return tuple(found)
+
+
+def latest_version() -> int:
+    migrations = discover()
+    return migrations[-1].version if migrations else 0
+
+
+def _ensure_migrations_table(conn: sqlite3.Connection) -> None:
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS schema_migrations ("
+        "version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)"
+    )
+
+
+def applied_versions(conn: sqlite3.Connection) -> tuple[int, ...]:
+    _ensure_migrations_table(conn)
+    rows = conn.execute("SELECT version FROM schema_migrations ORDER BY version").fetchall()
+    return tuple(int(row[0]) for row in rows)
+
+
+BASELINE_COLUMNS = frozenset({"company_fold", "employment_type", "source_category"})
+BASELINE_TABLES = frozenset({"coverage_classifications", "workday_crawls", "workday_crawl_seen"})
+
+
+def _refuse_a_pre_baseline_database(conn: sqlite3.Connection, db_path: Path) -> None:
+    columns = {str(row[1]) for row in conn.execute("PRAGMA table_info(jobs)").fetchall()}
+    tables = {
+        str(row[0])
+        for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
+    }
+    missing = sorted(BASELINE_COLUMNS - columns) + sorted(BASELINE_TABLES - tables)
+    if not missing:
+        return
+    raise SchemaVersionError(
+        f"database at {db_path} records the current schema version but is missing "
+        f"{', '.join(missing)} — it predates this baseline. Back it up or delete it, then run "
+        "stage sync; the corpus is reproducible in under a minute"
+    )
+
+
+def snapshot_path(db_path: Path, when: datetime) -> Path:
+    return db_path.with_name(f"{db_path.name}.bak-{when.strftime('%Y%m%dT%H%M%S')}")
+
+
+def _snapshot(conn: sqlite3.Connection, db_path: Path) -> Path:
+    target = snapshot_path(db_path, datetime.now(UTC))
+    backup = sqlite3.connect(target)
+    try:
+        conn.backup(backup)
+    finally:
+        backup.close()
+    return target
+
+
+def migrate(conn: sqlite3.Connection, db_path: Path) -> tuple[int, ...]:
+    migrations = discover()
+    known = {migration.version for migration in migrations}
+    applied = applied_versions(conn)
+
+    unknown = [version for version in applied if version not in known]
+    if unknown:
+        raise SchemaVersionError(
+            f"database at {db_path} has schema version {max(unknown)}, newer than this build "
+            f"understands ({latest_version()}) — upgrade stage-cli"
+        )
+
+    pending = [migration for migration in migrations if migration.version not in applied]
+    if not pending:
+        _refuse_a_pre_baseline_database(conn, db_path)
+        return ()
+
+    if applied:
+        _snapshot(conn, db_path)
+
+    for migration in pending:
+        try:
+            conn.executescript(f"BEGIN IMMEDIATE;\n{migration.read()}")
+            conn.execute(
+                "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
+                (migration.version, migration.name, datetime.now(UTC).isoformat()),
+            )
+        except Exception:
+            conn.rollback()
+            raise
+        conn.commit()
+
+    return tuple(migration.version for migration in pending)
+
+
+__all__ = [
+    "BASELINE_COLUMNS",
+    "BASELINE_TABLES",
+    "Migration",
+    "SchemaVersionError",
+    "applied_versions",
+    "discover",
+    "latest_version",
+    "migrate",
+    "snapshot_path",
+]

stage/storage/repository.py

@@ -0,0 +1,213 @@
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Protocol
+
+from stage.domain import (
+    CompanyVisit,
+    CoverageClassification,
+    DetailFetch,
+    HttpValidator,
+    IntegrityFinding,
+    IntegrityRepair,
+    Job,
+    JobFilters,
+    PurgeResult,
+    QuarantinedJob,
+    QuarantineFilters,
+    RateState,
+    SourceVisit,
+    SyncRun,
+    VolumePoint,
+    WorkdayCrawl,
+    WorkdayCrawlStep,
+    WorkdayFacet,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class SourceBatch:
+    source: str
+    run_started_at: datetime
+    jobs: tuple[Job, ...] = field(default_factory=tuple)
+    closable_boards: tuple[str, ...] = field(default_factory=tuple)
+    unchanged_boards: tuple[str, ...] = field(default_factory=tuple)
+    validators: tuple[HttpValidator, ...] = field(default_factory=tuple)
+    rate_state: tuple[RateState, ...] = field(default_factory=tuple)
+    workday_facets: tuple[WorkdayFacet, ...] = field(default_factory=tuple)
+    forgotten_facets: tuple[WorkdayFacet, ...] = field(default_factory=tuple)
+    workday_crawls: tuple[WorkdayCrawlStep, ...] = field(default_factory=tuple)
+    detail_fetches: tuple[DetailFetch, ...] = field(default_factory=tuple)
+    visits: tuple[CompanyVisit, ...] = field(default_factory=tuple)
+    quarantined: tuple[QuarantinedJob, ...] = field(default_factory=tuple)
+    resolve_duplicates: "Callable[[Sequence[Job], Sequence[Job]], Sequence[object]] | None" = None
+
+    closes_whole_source: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class SourceBatchResult:
+    source: str
+    fetched: int
+    added: int
+    updated: int
+    closed: int
+    touched: int = 0
+    quarantined: int = 0
+    duplicates: int = 0
+    stored: int = 0
+
+
+class Repository(Protocol):
+    def apply_source_batch(self, batch: SourceBatch) -> SourceBatchResult:
+        pass
+
+    def load_validators(self, source: str) -> Mapping[str, HttpValidator]:
+        pass
+
+    def load_rate_state(self) -> Mapping[str, RateState]:
+        pass
+
+    def clear_rate_state(self, bucket: str | None = None) -> int:
+        pass
+
+    def stale_members(self, source: str, before: datetime) -> list[SourceVisit]:
+        pass
+
+    def detail_queue(self, source: str, limit: int) -> list[str]:
+        pass
+
+    def detail_queue_size(self, source: str) -> int:
+        pass
+
+    def load_workday_facets(self) -> Mapping[tuple[str, str], WorkdayFacet]:
+        pass
+
+    def load_workday_crawls(self) -> Mapping[str, WorkdayCrawl]:
+        pass
+
+    def list_quarantined(self, filters: QuarantineFilters) -> list[QuarantinedJob]:
+        pass
+
+    def count_duplicates(self) -> int:
+        pass
+
+    def purge(self, now: datetime) -> PurgeResult:
+        pass
+
+    def close_orphan_boards(self, sources: Sequence[str], boards: Sequence[str]) -> int:
+        pass
+
+    def preview_purge(self, now: datetime) -> PurgeResult:
+        pass
+
+    def tombstone_count(self) -> int:
+        pass
+
+    def count_quarantined(self, filters: QuarantineFilters) -> int:
+        pass
+
+    def relabel_quarantine(self, entries: Sequence[QuarantinedJob]) -> int:
+        pass
+
+    def refresh_quarantine_locations(self, resolve: Callable[[str], tuple[str, str | None]]) -> int:
+        pass
+
+    def quarantine_reason_counts(self) -> dict[str, int]:
+        pass
+
+    def list_jobs(self, filters: JobFilters) -> list[Job]:
+        pass
+
+    def get_job(self, job_id: str) -> Job | None:
+        pass
+
+    def duplicates_of(self, job_id: str) -> list[Job]:
+        pass
+
+    def search_jobs(self, query: str, filters: JobFilters) -> list[Job]:
+        pass
+
+    def count_search(self, query: str, filters: JobFilters) -> int:
+        pass
+
+    def count_jobs(self, filters: JobFilters) -> int:
+        pass
+
+    def company_names(self) -> list[str]:
+        pass
+
+    def previous_sync_at(self) -> datetime | None:
+        pass
+
+    def requests_since(self, since: datetime) -> tuple[dict[str, int], bool]:
+        pass
+
+    def closed_among(self, job_ids: Sequence[str]) -> int:
+        pass
+
+    def board_counts(self) -> dict[str, int]:
+        pass
+
+    def company_counts(self) -> dict[str, dict[str, int]]:
+        pass
+
+    def quarantine_company_counts(self) -> dict[str, dict[str, int]]:
+        pass
+
+    def quarantine_company_reasons(self) -> dict[str, dict[str, int]]:
+        pass
+
+    def company_apply_urls(self, companies: Sequence[str]) -> dict[str, tuple[str, ...]]:
+        pass
+
+    def coverage_classifications(self) -> list[CoverageClassification]:
+        pass
+
+    def record_coverage_classification(self, entry: CoverageClassification) -> bool:
+        pass
+
+    def clear_coverage_classification(self, company: str) -> bool:
+        pass
+
+    def record_sync_run(self, run: SyncRun) -> None:
+        pass
+
+    def last_sync_at(self) -> datetime | None:
+        pass
+
+    def clear_validators(self, source: str | None = None) -> int:
+        pass
+
+    def cached_url_count(self) -> int:
+        pass
+
+    def volume_history(self, limit: int) -> Mapping[str, list[VolumePoint]]:
+        pass
+
+    def run_history(self, limit: int) -> list[SyncRun]:
+        pass
+
+    def all_visits(self) -> list[SourceVisit]:
+        pass
+
+    def repair_integrity(self) -> list[IntegrityRepair]:
+        pass
+
+    def integrity_findings(self) -> list[IntegrityFinding]:
+        pass
+
+    def composition(self, column: str) -> dict[str, int]:
+        pass
+
+    def stored_counts(self) -> dict[str, int]:
+        pass
+
+    def schema_version(self) -> int:
+        pass
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["Repository", "SourceBatch", "SourceBatchResult"]

stage/storage/search.py

@@ -0,0 +1,28 @@
+import re
+
+_TERM = re.compile(r"^[0-9a-z]+$")
+
+MAX_TERM_LENGTH = 48
+MAX_TERMS = 32
+
+FTS_COLUMN_WEIGHTS = (5.0, 10.0, 8.0, 8.0, 2.0, 1.0)
+
+
+def search_terms(query: str) -> tuple[str, ...]:
+    from stage.lexicon import fold
+
+    found = [term[:MAX_TERM_LENGTH] for term in fold(query).split() if _TERM.match(term)]
+    return tuple(found[:MAX_TERMS])
+
+
+def match_expression(terms: tuple[str, ...]) -> str:
+    return " ".join(f'"{term}"*' for term in terms)
+
+
+__all__ = [
+    "FTS_COLUMN_WEIGHTS",
+    "MAX_TERMS",
+    "MAX_TERM_LENGTH",
+    "match_expression",
+    "search_terms",
+]