stage-cli 1.0.0__py3-none-any.whl

stage/services/sync.py

@@ -0,0 +1,1257 @@
+import asyncio
+import random
+import time
+from collections.abc import AsyncIterator, Callable, Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from datetime import UTC, datetime, timedelta
+
+from stage.classify import (
+    classify_role,
+    resolve_eligibility,
+    screen_degree_scope,
+    screen_is_cs_role,
+    screen_is_internship,
+    screen_location,
+    to_quarantined,
+)
+from stage.dedup import resolve_duplicates
+from stage.domain import (
+    BucketPlan,
+    Company,
+    CompanyDeferred,
+    CompanyFailed,
+    CompanyFinished,
+    CompanyStarted,
+    CompanyUnchanged,
+    CompanyVisit,
+    DetailFetch,
+    HttpValidator,
+    Job,
+    PlannedRequest,
+    QuarantinedJob,
+    RateState,
+    RequestLogged,
+    Rotation,
+    RotationMember,
+    SourceBlocked,
+    SourceCapped,
+    SourceFailed,
+    SourceFinished,
+    SourceFresh,
+    SourceResting,
+    SourceRotated,
+    SourceRunStats,
+    SourceStarted,
+    SourceVisit,
+    SyncEvent,
+    SyncFinished,
+    SyncOutcome,
+    SyncRun,
+    SyncStarted,
+    UnroutableCompanies,
+    WorkdayCrawl,
+    WorkdayCrawlStep,
+    WorkdayFacet,
+    is_resting,
+    rotate,
+)
+from stage.http import (
+    CEILING_BACKSTOP,
+    HostBudget,
+    HostBudgetExceededError,
+    HttpClient,
+    HttpError,
+    RatePosture,
+    ValidatorCache,
+    resolve,
+)
+from stage.normalize import (
+    canonical_apply_url,
+    detect_language,
+    resolve_location,
+    resolve_term,
+)
+from stage.sources import (
+    Adapter,
+    FeedAdapter,
+    FetchResult,
+    adapter_for_platform,
+    get_adapters,
+    get_feeds,
+)
+from stage.sources.workday import WorkdayAdapter
+from stage.storage import AsyncRepository, SourceBatch, SourceBatchResult
+
+
+class NoSourcesSelectedError(Exception):
+    pass
+
+
+def _bucket_postures(
+    grouped: Mapping[str, tuple[Adapter, list[Company]]], feeds: Mapping[str, FeedAdapter]
+) -> dict[str, RatePosture]:
+    claims: list[tuple[tuple[str, ...], RatePosture]] = [
+        (
+            _adapter_buckets(adapter, companies),
+            resolve(
+                adapter.rate_profile, [company.rate_profile for company in companies]
+            ).sized_for(len(companies), _reserve_for(adapter)),
+        )
+        for adapter, companies in grouped.values()
+    ]
+    claims.extend(
+        (_bucket_keys(feed.hosts, feed.bucket_key), resolve(feed.rate_profile, []))
+        for feed in feeds.values()
+    )
+
+    postures: dict[str, RatePosture] = {}
+    for keys, posture in claims:
+        for bucket in keys:
+            stored = postures.get(bucket)
+            postures[bucket] = posture if stored is None else stored.strictest(posture)
+    return postures
+
+
+def _bucket_keys(hosts: frozenset[str], bucket_key: str) -> tuple[str, ...]:
+    return (bucket_key,) if bucket_key else tuple(sorted(hosts))
+
+
+def _adapter_buckets(adapter: Adapter, companies: Sequence[Company]) -> tuple[str, ...]:
+    if adapter.bucket_key:
+        return (adapter.bucket_key,)
+    return tuple(sorted(adapter.hosts_for(companies) or adapter.hosts))
+
+
+def _plan_bounds(
+    adapter: Adapter,
+    source: str,
+    companies: Sequence[Company],
+    page_budgets: Mapping[str, int],
+    detail_budget: int,
+) -> list[tuple[str, str, int, int]]:
+    members: dict[str, int] = {}
+    worst: dict[str, int] = {}
+    for company in companies:
+        for bucket in _adapter_buckets(adapter, [company]):
+            members[bucket] = members.get(bucket, 0) + 1
+            worst[bucket] = worst.get(bucket, 0) + page_budgets.get(company.registry_key, 0)
+    bounds = [(bucket, source, members[bucket], worst[bucket]) for bucket in sorted(members)]
+    if detail_budget and bounds:
+        bucket, name, count, total = bounds[0]
+        bounds[0] = (bucket, name, count, total + detail_budget)
+    return bounds
+
+
+def _reserve_for(adapter: Adapter) -> int:
+    return getattr(adapter, "retry_reserve", 0) + adapter.detail_budget
+
+
+DAILY_RUNS = 4
+
+
+async def _spent_today(repository: AsyncRepository, since: datetime) -> tuple[dict[str, int], bool]:
+    return await repository.requests_since(since)
+
+
+def _daily_allowance(posture: RatePosture, spent: int, has_history: bool, buckets: int = 1) -> int:
+    if not has_history:
+        return posture.max_requests_per_run
+    span = max(1, buckets)
+    cap = min(CEILING_BACKSTOP * 2, posture.max_requests_per_run * DAILY_RUNS) * span
+    return max(0, min(posture.max_requests_per_run, cap - spent))
+
+
+def _active_block(
+    rate_state: Mapping[str, RateState], buckets: Sequence[str], now: datetime
+) -> RateState | None:
+    for bucket in buckets:
+        state = rate_state.get(bucket)
+        if state is not None and state.is_blocked(now):
+            return state
+    return None
+
+
+def _group_by_adapter(
+    companies: Sequence[Company],
+) -> tuple[dict[str, tuple[Adapter, list[Company]]], list[Company]]:
+    grouped: dict[str, tuple[Adapter, list[Company]]] = {}
+    unroutable: list[Company] = []
+    for company in companies:
+        if not company.enabled:
+            continue
+        adapter = adapter_for_platform(company.platform)
+        if adapter is None:
+            unroutable.append(company)
+            continue
+        entry = grouped.setdefault(adapter.name, (adapter, []))
+        entry[1].append(company)
+    return grouped, unroutable
+
+
+def _registry_boards(
+    companies: Sequence[Company],
+) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    boards: set[str] = set()
+    unreadable: set[str] = set()
+    covered: set[str] = set()
+    for company in companies:
+        adapter = adapter_for_platform(company.platform)
+        if adapter is None:
+            continue
+        covered.add(adapter.name)
+        try:
+            boards.add(adapter.board_key(company))
+        except Exception:
+            unreadable.add(adapter.name)
+    return tuple(sorted(covered - unreadable)), tuple(sorted(boards))
+
+
+def _select(
+    companies: Sequence[Company],
+    sources: Sequence[str] | None,
+    excluded: Sequence[str] | None = None,
+) -> tuple[dict[str, tuple[Adapter, list[Company]]], dict[str, FeedAdapter], list[Company]]:
+    grouped, unroutable = _group_by_adapter(companies)
+    feeds = dict(get_feeds())
+    known = set(grouped) | set(feeds) | set(get_adapters())
+    if sources is not None:
+        selected = set(sources)
+        unknown = selected - known
+        if unknown:
+            raise NoSourcesSelectedError(
+                f"no enabled companies for source(s): {', '.join(sorted(unknown))}"
+            )
+        grouped = {name: value for name, value in grouped.items() if name in selected}
+        feeds = {name: value for name, value in feeds.items() if name in selected}
+    if excluded:
+        dropped = set(excluded)
+        unknown = dropped - known
+        if unknown:
+            raise NoSourcesSelectedError(
+                f"unknown source(s) to exclude: {', '.join(sorted(unknown))}"
+            )
+        grouped = {name: value for name, value in grouped.items() if name not in dropped}
+        feeds = {name: value for name, value in feeds.items() if name not in dropped}
+    if not grouped and not feeds:
+        detail = ""
+        if unroutable:
+            platforms = ", ".join(sorted({company.platform.value for company in unroutable}))
+            detail = f" ({len(unroutable)} enabled row(s) sit on unrouted platform(s): {platforms})"
+        raise NoSourcesSelectedError(
+            f"the registry has no enabled companies with a matching adapter{detail}"
+        )
+    return grouped, feeds, unroutable
+
+
+def normalize_batch(jobs: Sequence[Job]) -> tuple[tuple[Job, ...], tuple[QuarantinedJob, ...]]:
+    kept: list[Job] = []
+    rejected: list[QuarantinedJob] = []
+    for job in jobs:
+        location = resolve_location(job.location_raw)
+        term = resolve_term(
+            title=job.title_raw,
+            description=job.description,
+            structured_terms=job.signals.terms,
+            structured_season=job.signals.season,
+        )
+        role = classify_role(job.title_raw, job.description, job.signals.category)
+        language = detect_language(job.title_raw, job.description)
+        eligibility = resolve_eligibility(job)
+        normalized = replace(
+            job,
+            apply_url_canonical=canonical_apply_url(job.apply_url_raw),
+            location=location.bucket,
+            remote_scope=location.remote_scope,
+            term=term.term,
+            role=role.role,
+            language=language.language,
+            degree_requirement=eligibility.degree_requirement,
+            work_auth_flag=eligibility.work_auth_flag,
+        )
+        rejection = (
+            screen_is_internship(normalized)
+            or screen_location(normalized)
+            or screen_degree_scope(normalized)
+            or screen_is_cs_role(normalized)
+        )
+        if rejection is None:
+            kept.append(normalized)
+        else:
+            rejected.append(to_quarantined(normalized, rejection))
+    return tuple(kept), tuple(rejected)
+
+
+async def _fetch_company(
+    adapter: Adapter,
+    company: Company,
+    client: HttpClient,
+    now: datetime,
+    facets: Mapping[tuple[str, str], WorkdayFacet],
+    details: Sequence[str],
+    crawls: Mapping[str, WorkdayCrawl],
+    page_budget: int,
+    final_workday_pass: bool,
+) -> tuple[Company, FetchResult | None, str, float, bool, bool]:
+    started = time.perf_counter()
+    try:
+        if isinstance(adapter, WorkdayAdapter):
+            result = await adapter.fetch(
+                company,
+                client,
+                now,
+                facets,
+                details,
+                crawl=crawls.get(_safe_board_key(adapter, company)),
+                page_budget=page_budget,
+            )
+            crawl = result.workday_crawl
+            if (
+                final_workday_pass
+                and crawl is not None
+                and not crawl.complete
+                and not crawl.discard
+            ):
+                message = (
+                    "resumed once and still incomplete; postings stay open, crawl state discarded"
+                )
+                result = replace(
+                    result,
+                    degraded=f"{result.degraded}; {message}" if result.degraded else message,
+                    workday_crawl=replace(crawl, discard=True),
+                )
+        else:
+            result = await adapter.fetch(company, client, now, facets, details)
+    except HostBudgetExceededError as exc:
+        elapsed = (time.perf_counter() - started) * 1000
+        return company, None, f"{type(exc).__name__}: {exc}", elapsed, False, True
+    except Exception as exc:
+        elapsed = (time.perf_counter() - started) * 1000
+        unreachable = isinstance(exc, HttpError)
+        return company, None, f"{type(exc).__name__}: {exc}", elapsed, True, unreachable
+    return company, result, "", (time.perf_counter() - started) * 1000, True, False
+
+
+def _keepable_validators(cache: ValidatorCache, skip_urls: set[str]) -> tuple[HttpValidator, ...]:
+    return tuple(validator for url, validator in cache.pending.items() if url not in skip_urls)
+
+
+def _advance_cursor(
+    settled: Sequence[RateState], rotation: Rotation, bucket: str, now: datetime
+) -> tuple[RateState, ...]:
+    if not rotation.rotating:
+        return tuple(
+            state.with_cursor("", now)
+            if state.bucket == bucket and state.rotation_cursor
+            else state
+            for state in settled
+        )
+    updated = [
+        state.with_cursor(rotation.cursor, now) if state.bucket == bucket else state
+        for state in settled
+    ]
+    if not any(state.bucket == bucket for state in updated):
+        updated.append(RateState(bucket=bucket, updated_at=now, rotation_cursor=rotation.cursor))
+    return tuple(updated)
+
+
+def _bucket_plans(
+    bounds: Sequence[tuple[str, str, int, int]], postures: Mapping[str, RatePosture]
+) -> tuple[BucketPlan, ...]:
+    planned: dict[str, int] = {}
+    worst: dict[str, int] = {}
+    sources: dict[str, list[str]] = {}
+    for bucket, source, companies, worst_case in bounds:
+        planned[bucket] = planned.get(bucket, 0) + companies
+        worst[bucket] = worst.get(bucket, 0) + worst_case
+        sources.setdefault(bucket, []).append(source)
+    return tuple(
+        BucketPlan(
+            bucket=bucket,
+            sources=tuple(sorted(sources[bucket])),
+            planned=planned[bucket],
+            worst_case=worst[bucket],
+            ceiling=postures[bucket].max_requests_per_run
+            if bucket in postures
+            else RatePosture().max_requests_per_run,
+        )
+        for bucket in sorted(planned)
+    )
+
+
+def _safe_plan(adapter: Adapter, company: Company) -> tuple[tuple[str, ...], str]:
+    try:
+        return adapter.plan(company), ""
+    except Exception as exc:
+        return (), f"{type(exc).__name__}: {exc}"
+
+
+def _is_resting(
+    adapter: Adapter,
+    company: Company,
+    visits: Mapping[str, SourceVisit],
+    now: datetime,
+) -> bool:
+    visit = visits.get(_safe_board_key(adapter, company))
+    return visit is not None and is_resting(visit, now)
+
+
+def _partial_reason(
+    failed_sources: Sequence[str],
+    unroutable: Sequence[Company],
+    blocked_sources: Sequence[str],
+    deferred_sources: Sequence[str],
+) -> str:
+    reasons = []
+    if failed_sources:
+        reasons.append(f"{len(failed_sources)} source(s) failed")
+    if blocked_sources:
+        reasons.append(f"{len(blocked_sources)} source(s) blocked")
+    if deferred_sources:
+        reasons.append(f"{len(deferred_sources)} source(s) stopped on their host budget")
+    if unroutable:
+        reasons.append(f"{len(unroutable)} row(s) no adapter could route")
+    return "; ".join(reasons)
+
+
+def _stalest_first(
+    adapter: Adapter, companies: Sequence[Company], last_success: Mapping[str, datetime | None]
+) -> list[Company]:
+    def age(company: Company) -> tuple[int, datetime]:
+        seen = last_success.get(_safe_board_key(adapter, company))
+        return (1, seen) if seen is not None else (0, datetime.min.replace(tzinfo=UTC))
+
+    return sorted(companies, key=age)
+
+
+def _safe_board_key(adapter: Adapter, company: Company) -> str:
+    try:
+        return adapter.board_key(company)
+    except Exception:
+        return f"<unresolved>:{company.registry_key}"
+
+
+def _drain(client: HttpClient, source: str) -> list[RequestLogged]:
+    return [
+        RequestLogged(
+            source=source,
+            method=record.method,
+            url=record.url,
+            status=record.status,
+            elapsed_ms=record.elapsed_ms,
+            attempt=record.attempt,
+            error=record.error,
+        )
+        for record in client.drain_log()
+    ]
+
+
+@dataclass(slots=True)
+class FetchHarvest:
+    collected: list[Job] = field(default_factory=list)
+    closable: list[str] = field(default_factory=list)
+    unchanged: list[str] = field(default_factory=list)
+    skip_urls: set[str] = field(default_factory=set)
+    visits: list[CompanyVisit] = field(default_factory=list)
+    resolved_facets: list[WorkdayFacet] = field(default_factory=list)
+    forgotten_facets: list[WorkdayFacet] = field(default_factory=list)
+    workday_crawls: list[WorkdayCrawlStep] = field(default_factory=list)
+    detail_outcomes: list[DetailFetch] = field(default_factory=list)
+    errors: int = 0
+    deferred_boards: int = 0
+    succeeded: bool = False
+
+
+@dataclass(slots=True)
+class RunLedger:
+    stats: list[SourceRunStats] = field(default_factory=list)
+    failed: list[str] = field(default_factory=list)
+    succeeded: list[bool] = field(default_factory=list)
+    blocked: list[str] = field(default_factory=list)
+    deferred: list[str] = field(default_factory=list)
+    plan_bounds: list[tuple[str, str, int, int]] = field(default_factory=list)
+
+
+async def _merge(
+    streams: Sequence[tuple[str, AsyncIterator[SyncEvent]]],
+    ledger: "RunLedger",
+) -> AsyncIterator[SyncEvent]:
+    if len(streams) == 1:
+        source, stream = streams[0]
+        try:
+            async for event in stream:
+                yield event
+        except asyncio.CancelledError:
+            raise
+        except Exception as stream_exc:
+            error = f"{type(stream_exc).__name__}: {stream_exc}"
+            ledger.failed.append(source)
+            ledger.stats.append(SourceRunStats(source=source, errors=1))
+            yield SourceFailed(source=source, error=error)
+        return
+
+    queue: asyncio.Queue[SyncEvent | tuple[str, Exception] | None] = asyncio.Queue()
+
+    async def pump(source: str, stream: AsyncIterator[SyncEvent]) -> None:
+        try:
+            async for event in stream:
+                await queue.put(event)
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            await queue.put((source, exc))
+        finally:
+            queue.put_nowait(None)
+
+    tasks = [asyncio.create_task(pump(source, stream)) for source, stream in streams]
+    try:
+        done = 0
+        while done < len(tasks):
+            item = await queue.get()
+            if item is None:
+                done += 1
+            elif isinstance(item, tuple):
+                source, queued_exc = item
+                error = f"{type(queued_exc).__name__}: {queued_exc}"
+                ledger.failed.append(source)
+                ledger.stats.append(SourceRunStats(source=source, errors=1))
+                yield SourceFailed(source=source, error=error)
+            else:
+                yield item
+    finally:
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+
+async def _plan_company_source(
+    adapter: Adapter, companies: Sequence[Company], validators: dict[str, HttpValidator]
+) -> AsyncIterator[SyncEvent]:
+    for company in companies:
+        urls, error = _safe_plan(adapter, company)
+        if error:
+            yield CompanyFailed(
+                source=adapter.name, company=company.name, error=error, elapsed_ms=0.0
+            )
+            continue
+        for url in urls:
+            cached = validators.get(url)
+            has_validator = cached is not None and cached.usable
+            yield PlannedRequest(
+                source=adapter.name,
+                company=company.name,
+                url=url,
+                has_validator=has_validator,
+                expectation="likely 304 (validator on file)" if has_validator else "full response",
+            )
+
+
+async def _rehearse_company_source(
+    adapter: Adapter,
+    source_name: str,
+    ordered: Sequence[Company],
+    seed: dict[str, HttpValidator],
+    ledger: "RunLedger",
+    page_budgets: Mapping[str, int],
+    detail_budget: int,
+    source_clock: float,
+) -> AsyncIterator[SyncEvent]:
+    ledger.plan_bounds.extend(
+        _plan_bounds(adapter, source_name, ordered, page_budgets, detail_budget)
+    )
+    async for event in _plan_company_source(adapter, ordered, seed):
+        yield event
+    yield SourceFinished(
+        source=source_name,
+        fetched=0,
+        added=0,
+        updated=0,
+        closed=0,
+        failed_companies=0,
+        elapsed_ms=(time.perf_counter() - source_clock) * 1000,
+    )
+
+
+def _drop_recently_refreshed(
+    adapter: Adapter,
+    companies: Sequence[Company],
+    last_success: Mapping[str, datetime | None],
+    open_crawls: frozenset[str],
+    run_started_at: datetime,
+    window_h: float,
+    skip: bool,
+) -> tuple[list[Company], int]:
+    if skip or window_h <= 0:
+        return list(companies), 0
+    since = run_started_at - timedelta(hours=window_h)
+    kept = [
+        company
+        for company in companies
+        if _safe_board_key(adapter, company) in open_crawls
+        or (seen := last_success.get(_safe_board_key(adapter, company))) is None
+        or seen < since
+    ]
+    return kept, len(companies) - len(kept)
+
+
+def _drop_resting(
+    adapter: Adapter,
+    companies: Sequence[Company],
+    board_visits: Mapping[str, SourceVisit],
+    run_started_at: datetime,
+    skip: bool,
+) -> tuple[list[Company], int]:
+    if skip:
+        return list(companies), 0
+    awake = [
+        company
+        for company in companies
+        if not _is_resting(adapter, company, board_visits, run_started_at)
+    ]
+    return awake, len(companies) - len(awake)
+
+
+async def _persist_source_batch(
+    repository: AsyncRepository,
+    template: SourceBatch,
+    collected: Sequence[Job],
+) -> tuple[SourceBatchResult, float, float]:
+    normalize_clock = time.perf_counter()
+    kept, rejected = await asyncio.to_thread(normalize_batch, list(collected))
+    normalize_ms = (time.perf_counter() - normalize_clock) * 1000
+    write_clock = time.perf_counter()
+    counts = await repository.apply_source_batch(replace(template, jobs=kept, quarantined=rejected))
+    return counts, normalize_ms, (time.perf_counter() - write_clock) * 1000
+
+
+async def _run_company_source(
+    repository: AsyncRepository,
+    adapter: Adapter,
+    companies: Sequence[Company],
+    run_started_at: datetime,
+    shuffler: random.Random,
+    dry_run: bool,
+    ledger: "RunLedger",
+    rate_state: Mapping[str, RateState],
+    budgets: dict[str, HostBudget],
+    postures: Mapping[str, RatePosture],
+    force_refresh: bool,
+    spent_today: Mapping[str, int],
+    has_history: bool,
+) -> AsyncIterator[SyncEvent]:
+    source_name = adapter.name
+    buckets = _adapter_buckets(adapter, companies)
+    block = _active_block(rate_state, buckets, run_started_at)
+    if block is not None:
+        ledger.blocked.append(source_name)
+        if not dry_run:
+            ledger.stats.append(SourceRunStats(source=source_name, blocked=True))
+        yield SourceBlocked(
+            source=source_name,
+            bucket=block.bucket,
+            blocked_until=block.blocked_until or run_started_at,
+            remaining_s=block.blocks_remaining_s(run_started_at),
+            reason=block.reason,
+            consecutive_failures=block.consecutive_failures,
+        )
+        return
+
+    rotation_bucket = buckets[0]
+    stored = rate_state.get(rotation_bucket)
+    workday_adapter = adapter if isinstance(adapter, WorkdayAdapter) else None
+    is_workday = workday_adapter is not None
+    window_h = postures.get(rotation_bucket, RatePosture()).refresh_interval_h
+    board_visits = {
+        visit.board: visit for visit in await repository.all_visits() if visit.source == source_name
+    }
+    last_success = {board: visit.last_success_at for board, visit in board_visits.items()}
+    rotation = rotate(
+        [RotationMember(key=company.registry_key) for company in companies],
+        cursor=stored.rotation_cursor if stored is not None else "",
+        budget=adapter.rotation_slice,
+    )
+    covered = set(rotation.selected)
+    ordered = [company for company in companies if company.registry_key in covered]
+    source_clock = time.perf_counter()
+    fetch_clock = time.perf_counter()
+
+    open_crawls = (
+        frozenset(await repository.load_workday_crawls())
+        if workday_adapter is not None
+        else frozenset()
+    )
+    skip_refresh = force_refresh or dry_run
+    ordered, refreshed_recently = _drop_recently_refreshed(
+        adapter, ordered, last_success, open_crawls, run_started_at, window_h, skip_refresh
+    )
+    ordered, resting_boards = _drop_resting(
+        adapter, ordered, board_visits, run_started_at, skip_refresh
+    )
+
+    seed = dict(await repository.load_validators(source_name))
+    resuming_boards: frozenset[str] = frozenset()
+    base = postures.get(rotation_bucket) or resolve(
+        adapter.rate_profile, [company.rate_profile for company in ordered]
+    )
+    allowance = _daily_allowance(
+        base, spent_today.get(source_name, 0), has_history, buckets=len(buckets)
+    )
+    posture = replace(base, max_requests_per_run=allowance)
+    if workday_adapter is not None:
+        facets = await repository.load_workday_facets()
+        crawls = await repository.load_workday_crawls()
+        resuming_boards = frozenset(crawls)
+        page_budgets, detail_budget = (
+            workday_adapter.crawl_budgets(ordered, crawls, facets, posture.max_requests_per_run)
+            if ordered
+            else ({}, 0)
+        )
+    else:
+        page_budgets = {
+            company.registry_key: adapter.max_requests_per_company for company in ordered
+        }
+        detail_budget = adapter.detail_budget
+        facets = {}
+        crawls = {}
+    shuffler.shuffle(ordered)
+    ordered = _stalest_first(adapter, ordered, last_success)
+    yield SourceStarted(source=source_name, companies=len(ordered))
+    if allowance < base.max_requests_per_run:
+        yield SourceCapped(
+            source=source_name,
+            bucket=rotation_bucket,
+            spent=spent_today.get(source_name, 0),
+            allowance=allowance,
+            ceiling=base.max_requests_per_run,
+        )
+    if refreshed_recently:
+        yield SourceFresh(
+            source=source_name,
+            skipped=refreshed_recently,
+            remaining=len(ordered),
+            refresh_interval_h=window_h,
+        )
+    if resting_boards:
+        yield SourceResting(source=source_name, skipped=resting_boards, remaining=len(ordered))
+    if rotation.rotating:
+        yield SourceRotated(
+            source=source_name,
+            bucket=rotation_bucket,
+            selected=len(rotation.selected),
+            deferred=len(rotation.deferred),
+            cursor=rotation.cursor,
+            wrapped=rotation.wrapped,
+        )
+    detail_queue = await repository.detail_queue(source_name, detail_budget)
+
+    if dry_run:
+        async for event in _rehearse_company_source(
+            adapter,
+            source_name,
+            ordered,
+            seed,
+            ledger,
+            page_budgets,
+            detail_budget if is_workday else 0,
+            source_clock,
+        ):
+            yield event
+        return
+
+    cache = ValidatorCache(seed)
+    harvest = FetchHarvest()
+
+    async with HttpClient(
+        allowed_hosts=adapter.hosts_for(ordered),
+        posture=posture,
+        cache=cache,
+        rng=shuffler,
+        bucket_key=adapter.bucket_key,
+        rate_state=rate_state,
+        now=run_started_at,
+        budgets=budgets,
+        postures={**postures, rotation_bucket: posture},
+    ) as client:
+        pending = [
+            _fetch_company(
+                adapter,
+                company,
+                client,
+                run_started_at,
+                facets,
+                detail_queue,
+                crawls,
+                page_budgets[company.registry_key],
+                _safe_board_key(adapter, company) in resuming_boards,
+            )
+            for company in ordered
+        ]
+        for company in ordered:
+            yield CompanyStarted(source=source_name, company=company.name)
+
+        for finished in asyncio.as_completed(pending):
+            company, result, error, elapsed, attempted, unreachable = await finished
+            for record in _drain(client, source_name):
+                yield record
+            board = _safe_board_key(adapter, company)
+            if result is None and not attempted:
+                harvest.skip_urls.update(_safe_plan(adapter, company)[0])
+                harvest.deferred_boards += 1
+                yield CompanyDeferred(source=source_name, company=company.name, reason=error)
+            elif result is None:
+                harvest.errors += 1
+                harvest.skip_urls.update(_safe_plan(adapter, company)[0])
+                harvest.visits.append(
+                    CompanyVisit(board=board, succeeded=False, error=error, label=company.name)
+                )
+                yield CompanyFailed(
+                    source=source_name,
+                    company=company.name,
+                    error=error,
+                    elapsed_ms=elapsed,
+                    unreachable=unreachable,
+                )
+            elif result.not_modified:
+                harvest.visits.append(CompanyVisit(board=board, succeeded=True, label=company.name))
+                harvest.unchanged.append(board)
+                yield CompanyUnchanged(source=source_name, company=company.name, elapsed_ms=elapsed)
+            else:
+                harvest.succeeded = True
+                harvest.visits.append(CompanyVisit(board=board, succeeded=True, label=company.name))
+                harvest.collected.extend(result.jobs)
+                harvest.resolved_facets.extend(
+                    entry for entry in result.facets if isinstance(entry, WorkdayFacet)
+                )
+                harvest.forgotten_facets.extend(
+                    entry for entry in result.forgotten_facets if isinstance(entry, WorkdayFacet)
+                )
+                harvest.detail_outcomes.extend(
+                    entry for entry in result.detail_fetches if isinstance(entry, DetailFetch)
+                )
+                if isinstance(result.workday_crawl, WorkdayCrawlStep):
+                    harvest.workday_crawls.append(result.workday_crawl)
+                if result.authoritative:
+                    harvest.closable.append(board)
+                if result.degraded:
+                    harvest.skip_urls.update(_safe_plan(adapter, company)[0])
+                    harvest.skip_urls.update(result.stale_urls)
+                yield CompanyFinished(
+                    source=source_name,
+                    company=company.name,
+                    fetched=len(result.jobs),
+                    elapsed_ms=elapsed,
+                    degraded=result.degraded,
+                )
+
+        metrics = _client_metrics(client)
+        validators = _keepable_validators(cache, harvest.skip_urls)
+        settled = _advance_cursor(
+            client.rate_state(run_started_at), rotation, rotation_bucket, run_started_at
+        )
+
+    fetch_ms = (time.perf_counter() - fetch_clock) * 1000
+    counts, normalize_ms, write_ms = await _persist_source_batch(
+        repository,
+        SourceBatch(
+            source=source_name,
+            run_started_at=run_started_at,
+            jobs=(),
+            closable_boards=tuple(harvest.closable),
+            unchanged_boards=tuple(harvest.unchanged),
+            validators=validators,
+            rate_state=settled,
+            workday_facets=tuple(harvest.resolved_facets),
+            forgotten_facets=tuple(harvest.forgotten_facets),
+            workday_crawls=tuple(harvest.workday_crawls),
+            detail_fetches=tuple(harvest.detail_outcomes),
+            visits=tuple(harvest.visits),
+            resolve_duplicates=resolve_duplicates,
+        ),
+        harvest.collected,
+    )
+    if harvest.succeeded or harvest.unchanged:
+        ledger.succeeded.append(True)
+    if harvest.errors:
+        ledger.failed.append(source_name)
+    if harvest.deferred_boards:
+        ledger.deferred.append(source_name)
+
+    elapsed_ms = (time.perf_counter() - source_clock) * 1000
+    ledger.stats.append(
+        replace(
+            _stats(source_name, counts, harvest.errors, metrics, elapsed_ms),
+            deferred=len(rotation.deferred) + harvest.deferred_boards,
+        )
+    )
+    yield _finished(
+        source_name,
+        counts,
+        harvest.errors,
+        metrics,
+        elapsed_ms,
+        fetch_ms=fetch_ms,
+        normalize_ms=normalize_ms,
+        write_ms=write_ms,
+    )
+
+
+async def _run_feed_source(
+    repository: AsyncRepository,
+    feed: FeedAdapter,
+    run_started_at: datetime,
+    shuffler: random.Random,
+    dry_run: bool,
+    ledger: "RunLedger",
+    rate_state: Mapping[str, RateState],
+    budgets: dict[str, HostBudget],
+    postures: Mapping[str, RatePosture],
+) -> AsyncIterator[SyncEvent]:
+    source_name = feed.name
+    block = _active_block(rate_state, _bucket_keys(feed.hosts, feed.bucket_key), run_started_at)
+    if block is not None:
+        ledger.blocked.append(source_name)
+        if not dry_run:
+            ledger.stats.append(SourceRunStats(source=source_name, blocked=True))
+        yield SourceBlocked(
+            source=source_name,
+            bucket=block.bucket,
+            blocked_until=block.blocked_until or run_started_at,
+            remaining_s=block.blocks_remaining_s(run_started_at),
+            reason=block.reason,
+            consecutive_failures=block.consecutive_failures,
+        )
+        return
+
+    source_clock = time.perf_counter()
+    yield SourceStarted(source=source_name, companies=0)
+
+    fetch_clock = time.perf_counter()
+    seed = dict(await repository.load_validators(source_name))
+
+    if dry_run:
+        planned_urls = feed.plan(run_started_at)
+        for bucket in _bucket_keys(feed.hosts, feed.bucket_key):
+            ledger.plan_bounds.append((bucket, source_name, len(planned_urls), len(planned_urls)))
+        for url in planned_urls:
+            cached = seed.get(url)
+            has_validator = cached is not None and cached.usable
+            yield PlannedRequest(
+                source=source_name,
+                company=f"season {feed.season_year(run_started_at)}",
+                url=url,
+                has_validator=has_validator,
+                expectation="likely 304 (validator on file)" if has_validator else "full response",
+            )
+        yield SourceFinished(
+            source=source_name,
+            fetched=0,
+            added=0,
+            updated=0,
+            closed=0,
+            failed_companies=0,
+            elapsed_ms=(time.perf_counter() - source_clock) * 1000,
+        )
+        return
+
+    cache = ValidatorCache(seed)
+    collected: list[Job] = []
+    errors = 0
+    unchanged = False
+    degraded = ""
+    authoritative = True
+    stale_urls: tuple[str, ...] = ()
+    label = f"{source_name} {feed.season_year(run_started_at)}"
+
+    async with HttpClient(
+        allowed_hosts=feed.hosts,
+        posture=resolve(feed.rate_profile, []),
+        cache=cache,
+        rng=shuffler,
+        bucket_key=feed.bucket_key,
+        rate_state=rate_state,
+        now=run_started_at,
+        budgets=budgets,
+        postures=postures,
+    ) as client:
+        yield CompanyStarted(source=source_name, company=label)
+        started = time.perf_counter()
+        try:
+            result = await feed.fetch(client, run_started_at)
+        except Exception as exc:
+            errors = 1
+            result = None
+            error_text = f"{type(exc).__name__}: {exc}"
+        elapsed = (time.perf_counter() - started) * 1000
+
+        for record in _drain(client, source_name):
+            yield record
+
+        if result is None:
+            yield CompanyFailed(
+                source=source_name, company=label, error=error_text, elapsed_ms=elapsed
+            )
+        elif result.not_modified:
+            unchanged = True
+            yield CompanyUnchanged(source=source_name, company=label, elapsed_ms=elapsed)
+        else:
+            collected.extend(result.jobs)
+            degraded = result.degraded
+            authoritative = result.authoritative
+            stale_urls = result.stale_urls
+            yield CompanyFinished(
+                source=source_name,
+                company=label,
+                fetched=len(result.jobs),
+                elapsed_ms=elapsed,
+                degraded=result.degraded,
+            )
+
+        metrics = _client_metrics(client)
+        incomplete = bool(errors) or bool(degraded)
+        skip_urls = set(feed.plan(run_started_at)) | set(stale_urls) if incomplete else set()
+        validators = _keepable_validators(cache, skip_urls)
+        settled = client.rate_state(run_started_at)
+
+    fetch_ms = (time.perf_counter() - fetch_clock) * 1000
+    counts, normalize_ms, write_ms = await _persist_source_batch(
+        repository,
+        SourceBatch(
+            source=source_name,
+            run_started_at=run_started_at,
+            jobs=(),
+            validators=validators,
+            rate_state=settled,
+            resolve_duplicates=resolve_duplicates,
+            closes_whole_source=authoritative and not incomplete and not unchanged,
+            visits=(
+                CompanyVisit(
+                    board=source_name,
+                    succeeded=not errors,
+                    error=error_text if errors else "",
+                    label=label,
+                ),
+            ),
+        ),
+        collected,
+    )
+    if not errors:
+        ledger.succeeded.append(True)
+    else:
+        ledger.failed.append(source_name)
+
+    elapsed_ms = (time.perf_counter() - source_clock) * 1000
+    ledger.stats.append(_stats(source_name, counts, errors, metrics, elapsed_ms))
+    yield _finished(
+        source_name,
+        counts,
+        errors,
+        metrics,
+        elapsed_ms,
+        fetch_ms=fetch_ms,
+        normalize_ms=normalize_ms,
+        write_ms=write_ms,
+    )
+
+
+def _client_metrics(client: HttpClient) -> tuple[int, int, int, int, float, float]:
+    p50, p95 = client.latency_percentiles()
+    return (
+        client.request_count,
+        client.not_modified_count,
+        client.retry_count,
+        client.tightening_count,
+        p50,
+        p95,
+    )
+
+
+def _stats(
+    source: str,
+    counts: SourceBatchResult,
+    errors: int,
+    metrics: tuple[int, int, int, int, float, float],
+    elapsed_ms: float,
+) -> SourceRunStats:
+    requests, not_modified, retries, tightenings, p50, p95 = metrics
+    return SourceRunStats(
+        source=source,
+        fetched=counts.fetched,
+        added=counts.added,
+        updated=counts.updated,
+        closed=counts.closed,
+        quarantined=counts.quarantined,
+        errors=errors,
+        requests=requests,
+        not_modified=not_modified,
+        retries=retries,
+        tightenings=tightenings,
+        latency_p50_ms=p50,
+        latency_p95_ms=p95,
+        elapsed_ms=elapsed_ms,
+        stored=counts.stored,
+    )
+
+
+def _finished(
+    source: str,
+    counts: SourceBatchResult,
+    errors: int,
+    metrics: tuple[int, int, int, int, float, float],
+    elapsed_ms: float,
+    fetch_ms: float = 0.0,
+    normalize_ms: float = 0.0,
+    write_ms: float = 0.0,
+) -> SourceFinished:
+    requests, not_modified, retries, tightenings, p50, p95 = metrics
+    return SourceFinished(
+        source=source,
+        fetched=counts.fetched,
+        added=counts.added,
+        updated=counts.updated,
+        closed=counts.closed,
+        quarantined=counts.quarantined,
+        failed_companies=errors,
+        elapsed_ms=elapsed_ms,
+        fetch_ms=fetch_ms,
+        normalize_ms=normalize_ms,
+        write_ms=write_ms,
+        requests=requests,
+        not_modified=not_modified,
+        retries=retries,
+        tightenings=tightenings,
+        latency_p50_ms=p50,
+        latency_p95_ms=p95,
+    )
+
+
+async def sync(
+    repository: AsyncRepository,
+    companies: Sequence[Company],
+    *,
+    sources: Sequence[str] | None = None,
+    excluded: Sequence[str] | None = None,
+    dry_run: bool = False,
+    force_refresh: bool = False,
+    now_fn: Callable[[], datetime] = lambda: datetime.now(UTC),
+    rng: random.Random | None = None,
+) -> AsyncIterator[SyncEvent]:
+    grouped, feeds, unroutable = _select(companies, sources, excluded)
+    shuffler = rng or random.Random()
+
+    run_started_at = now_fn()
+    run_clock = time.perf_counter()
+    yield SyncStarted(
+        sources=tuple(sorted(set(grouped) | set(feeds))),
+        companies=sum(len(entry[1]) for entry in grouped.values()),
+        started_at=run_started_at,
+    )
+    if unroutable:
+        yield UnroutableCompanies(
+            companies=tuple(company.name for company in unroutable),
+            platforms=tuple(sorted({company.platform.value for company in unroutable})),
+        )
+
+    ledger = RunLedger()
+
+    rate_state = await repository.load_rate_state()
+    spent_today, has_history = await _spent_today(repository, run_started_at - timedelta(hours=24))
+    budgets: dict[str, HostBudget] = {}
+    postures = _bucket_postures(grouped, feeds)
+
+    streams: list[tuple[str, AsyncIterator[SyncEvent]]] = [
+        (
+            name,
+            _run_company_source(
+                repository,
+                grouped[name][0],
+                grouped[name][1],
+                run_started_at,
+                shuffler,
+                dry_run,
+                ledger,
+                rate_state,
+                budgets,
+                postures,
+                force_refresh,
+                spent_today,
+                has_history,
+            ),
+        )
+        for name in sorted(grouped)
+    ]
+    streams.extend(
+        (
+            name,
+            _run_feed_source(
+                repository,
+                feeds[name],
+                run_started_at,
+                shuffler,
+                dry_run,
+                ledger,
+                rate_state,
+                budgets,
+                postures,
+            ),
+        )
+        for name in sorted(feeds)
+    )
+
+    async for event in _merge(streams, ledger):
+        yield event
+
+    if dry_run:
+        for event in _bucket_plans(ledger.plan_bounds, postures):
+            yield event
+        yield SyncFinished(
+            outcome=(
+                SyncOutcome.PARTIAL
+                if ledger.failed or unroutable or ledger.blocked
+                else SyncOutcome.SUCCESS
+            ),
+            added=0,
+            updated=0,
+            closed=0,
+            failed_sources=tuple(ledger.failed),
+            elapsed_ms=(time.perf_counter() - run_clock) * 1000,
+            dry_run=True,
+        )
+        return
+
+    if not ledger.succeeded and (ledger.failed or ledger.blocked):
+        outcome_status = SyncOutcome.FAILURE
+    elif ledger.failed or unroutable or ledger.blocked or ledger.deferred:
+        outcome_status = SyncOutcome.PARTIAL
+    else:
+        outcome_status = SyncOutcome.SUCCESS
+    partial_reason = (
+        _partial_reason(ledger.failed, unroutable, ledger.blocked, ledger.deferred)
+        if outcome_status is not SyncOutcome.SUCCESS
+        else ""
+    )
+
+    sources_with_boards, known_boards = _registry_boards(companies)
+    await repository.close_orphan_boards(sources_with_boards, known_boards)
+
+    purged = await repository.purge(now_fn())
+
+    await repository.record_sync_run(
+        SyncRun(
+            started_at=run_started_at,
+            finished_at=now_fn(),
+            outcome=outcome_status,
+            sources=tuple(ledger.stats),
+        )
+    )
+    yield SyncFinished(
+        outcome=outcome_status,
+        added=sum(item.added for item in ledger.stats),
+        updated=sum(item.updated for item in ledger.stats),
+        closed=sum(item.closed for item in ledger.stats),
+        quarantined=sum(item.quarantined for item in ledger.stats),
+        purged=purged.purged,
+        failed_sources=tuple(ledger.failed),
+        elapsed_ms=(time.perf_counter() - run_clock) * 1000,
+        requests=sum(item.requests for item in ledger.stats),
+        not_modified=sum(item.not_modified for item in ledger.stats),
+        partial_reason=partial_reason,
+    )