stage-cli 1.0.0__py3-none-any.whl

stage/storage/writer.py

@@ -0,0 +1,249 @@
+import asyncio
+import threading
+from collections.abc import Callable, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from pathlib import Path
+from types import TracebackType
+from typing import Self, TypeVar
+
+from stage.domain import (
+    CoverageClassification,
+    HttpValidator,
+    IntegrityFinding,
+    IntegrityRepair,
+    Job,
+    JobFilters,
+    PurgeResult,
+    QuarantinedJob,
+    QuarantineFilters,
+    RateState,
+    SourceVisit,
+    SyncRun,
+    VolumePoint,
+    WorkdayCrawl,
+    WorkdayFacet,
+)
+from stage.storage.repository import Repository, SourceBatch, SourceBatchResult
+from stage.storage.sqlite_repo import SqliteRepository
+
+T = TypeVar("T")
+
+
+class WriterNotStartedError(RuntimeError):
+    pass
+
+
+class DatabaseWriter:
+    def __init__(self, db_path: Path) -> None:
+        self._db_path = db_path
+        self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="stage-writer")
+        self._repository: Repository | None = None
+        self._thread_id: int | None = None
+
+    @property
+    def thread_id(self) -> int | None:
+        return self._thread_id
+
+    async def _submit(self, fn: Callable[[], T]) -> T:
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(self._executor, fn)
+
+    def _open(self) -> None:
+        self._thread_id = threading.get_ident()
+        self._repository = SqliteRepository.connect(self._db_path)
+
+    def _shutdown(self) -> None:
+        if self._repository is not None:
+            self._repository.close()
+            self._repository = None
+
+    async def start(self) -> Self:
+        await self._submit(self._open)
+        return self
+
+    async def aclose(self) -> None:
+        await self._submit(self._shutdown)
+        self._executor.shutdown(wait=True)
+
+    async def __aenter__(self) -> Self:
+        return await self.start()
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.aclose()
+
+    async def run(self, fn: Callable[[Repository], T]) -> T:
+        def call() -> T:
+            repository = self._repository
+            if repository is None:
+                raise WriterNotStartedError("DatabaseWriter.start() has not been awaited")
+            return fn(repository)
+
+        return await self._submit(call)
+
+
+class AsyncRepository:
+    def __init__(self, writer: DatabaseWriter) -> None:
+        self._writer = writer
+
+    async def apply_source_batch(self, batch: SourceBatch) -> SourceBatchResult:
+        return await self._writer.run(lambda repository: repository.apply_source_batch(batch))
+
+    async def load_validators(self, source: str) -> Mapping[str, HttpValidator]:
+        return await self._writer.run(lambda repository: repository.load_validators(source))
+
+    async def load_rate_state(self) -> Mapping[str, RateState]:
+        return await self._writer.run(lambda repository: repository.load_rate_state())
+
+    async def clear_rate_state(self, bucket: str | None = None) -> int:
+        return await self._writer.run(lambda repository: repository.clear_rate_state(bucket))
+
+    async def stale_members(self, source: str, before: datetime) -> list[SourceVisit]:
+        return await self._writer.run(lambda repository: repository.stale_members(source, before))
+
+    async def detail_queue(self, source: str, limit: int) -> list[str]:
+        return await self._writer.run(lambda repository: repository.detail_queue(source, limit))
+
+    async def detail_queue_size(self, source: str) -> int:
+        return await self._writer.run(lambda repository: repository.detail_queue_size(source))
+
+    async def load_workday_facets(self) -> Mapping[tuple[str, str], WorkdayFacet]:
+        return await self._writer.run(lambda repository: repository.load_workday_facets())
+
+    async def load_workday_crawls(self) -> Mapping[str, WorkdayCrawl]:
+        return await self._writer.run(lambda repository: repository.load_workday_crawls())
+
+    async def clear_validators(self, source: str | None = None) -> int:
+        return await self._writer.run(lambda repository: repository.clear_validators(source))
+
+    async def cached_url_count(self) -> int:
+        return await self._writer.run(lambda repository: repository.cached_url_count())
+
+    async def list_quarantined(self, filters: QuarantineFilters) -> list[QuarantinedJob]:
+        return await self._writer.run(lambda repository: repository.list_quarantined(filters))
+
+    async def count_quarantined(self, filters: QuarantineFilters) -> int:
+        return await self._writer.run(lambda repository: repository.count_quarantined(filters))
+
+    async def quarantine_reason_counts(self) -> dict[str, int]:
+        return await self._writer.run(lambda repository: repository.quarantine_reason_counts())
+
+    async def count_duplicates(self) -> int:
+        return await self._writer.run(lambda repository: repository.count_duplicates())
+
+    async def relabel_quarantine(self, entries: Sequence[QuarantinedJob]) -> int:
+        return await self._writer.run(lambda repository: repository.relabel_quarantine(entries))
+
+    async def refresh_quarantine_locations(
+        self, resolve: Callable[[str], tuple[str, str | None]]
+    ) -> int:
+        return await self._writer.run(
+            lambda repository: repository.refresh_quarantine_locations(resolve)
+        )
+
+    async def repair_integrity(self) -> list[IntegrityRepair]:
+        return await self._writer.run(lambda repository: repository.repair_integrity())
+
+    async def purge(self, now: datetime) -> PurgeResult:
+        return await self._writer.run(lambda repository: repository.purge(now))
+
+    async def close_orphan_boards(self, sources: Sequence[str], boards: Sequence[str]) -> int:
+        return await self._writer.run(
+            lambda repository: repository.close_orphan_boards(sources, boards)
+        )
+
+    async def preview_purge(self, now: datetime) -> PurgeResult:
+        return await self._writer.run(lambda repository: repository.preview_purge(now))
+
+    async def tombstone_count(self) -> int:
+        return await self._writer.run(lambda repository: repository.tombstone_count())
+
+    async def list_jobs(self, filters: JobFilters) -> list[Job]:
+        return await self._writer.run(lambda repository: repository.list_jobs(filters))
+
+    async def count_jobs(self, filters: JobFilters) -> int:
+        return await self._writer.run(lambda repository: repository.count_jobs(filters))
+
+    async def closed_among(self, job_ids: Sequence[str]) -> int:
+        return await self._writer.run(lambda repository: repository.closed_among(job_ids))
+
+    async def requests_since(self, since: datetime) -> tuple[dict[str, int], bool]:
+        return await self._writer.run(lambda repository: repository.requests_since(since))
+
+    async def previous_sync_at(self) -> datetime | None:
+        return await self._writer.run(lambda repository: repository.previous_sync_at())
+
+    async def company_names(self) -> list[str]:
+        return await self._writer.run(lambda repository: repository.company_names())
+
+    async def get_job(self, job_id: str) -> Job | None:
+        return await self._writer.run(lambda repository: repository.get_job(job_id))
+
+    async def duplicates_of(self, job_id: str) -> list[Job]:
+        return await self._writer.run(lambda repository: repository.duplicates_of(job_id))
+
+    async def search_jobs(self, query: str, filters: JobFilters) -> list[Job]:
+        return await self._writer.run(lambda repository: repository.search_jobs(query, filters))
+
+    async def count_search(self, query: str, filters: JobFilters) -> int:
+        return await self._writer.run(lambda repository: repository.count_search(query, filters))
+
+    async def board_counts(self) -> dict[str, int]:
+        return await self._writer.run(lambda repository: repository.board_counts())
+
+    async def company_counts(self) -> dict[str, dict[str, int]]:
+        return await self._writer.run(lambda repository: repository.company_counts())
+
+    async def quarantine_company_counts(self) -> dict[str, dict[str, int]]:
+        return await self._writer.run(lambda repository: repository.quarantine_company_counts())
+
+    async def quarantine_company_reasons(self) -> dict[str, dict[str, int]]:
+        return await self._writer.run(lambda repository: repository.quarantine_company_reasons())
+
+    async def company_apply_urls(self, companies: Sequence[str]) -> dict[str, tuple[str, ...]]:
+        return await self._writer.run(lambda repository: repository.company_apply_urls(companies))
+
+    async def coverage_classifications(self) -> list[CoverageClassification]:
+        return await self._writer.run(lambda repository: repository.coverage_classifications())
+
+    async def record_coverage_classification(self, entry: CoverageClassification) -> bool:
+        return await self._writer.run(
+            lambda repository: repository.record_coverage_classification(entry)
+        )
+
+    async def clear_coverage_classification(self, company: str) -> bool:
+        return await self._writer.run(
+            lambda repository: repository.clear_coverage_classification(company)
+        )
+
+    async def record_sync_run(self, run: SyncRun) -> None:
+        await self._writer.run(lambda repository: repository.record_sync_run(run))
+
+    async def last_sync_at(self) -> datetime | None:
+        return await self._writer.run(lambda repository: repository.last_sync_at())
+
+    async def volume_history(self, limit: int) -> Mapping[str, list[VolumePoint]]:
+        return await self._writer.run(lambda repository: repository.volume_history(limit))
+
+    async def run_history(self, limit: int) -> list[SyncRun]:
+        return await self._writer.run(lambda repository: repository.run_history(limit))
+
+    async def all_visits(self) -> list[SourceVisit]:
+        return await self._writer.run(lambda repository: repository.all_visits())
+
+    async def integrity_findings(self) -> list[IntegrityFinding]:
+        return await self._writer.run(lambda repository: repository.integrity_findings())
+
+    async def composition(self, column: str) -> dict[str, int]:
+        return await self._writer.run(lambda repository: repository.composition(column))
+
+    async def stored_counts(self) -> dict[str, int]:
+        return await self._writer.run(lambda repository: repository.stored_counts())
+
+    async def schema_version(self) -> int:
+        return await self._writer.run(lambda repository: repository.schema_version())

stage/tui/app.py

@@ -0,0 +1,82 @@
+from contextlib import AsyncExitStack
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from textual.app import App
+from textual.binding import Binding
+
+if TYPE_CHECKING:
+    from stage.storage import AsyncRepository
+
+from stage.tui.screens.postings import PostingsScreen
+from stage.tui.screens.splash import SplashScreen
+
+THEME_PATH = Path(__file__).parent / "theme.tcss"
+
+
+class StageApp(App[None]):
+    CSS_PATH = THEME_PATH
+    TITLE = "Stage"
+    ALLOW_MAXIMIZE = False
+
+    BINDINGS = [
+        Binding("q", "quit", "quit"),
+        Binding("ctrl+t", "next_theme", "theme", show=False),
+    ]
+
+    def __init__(self, database: Path, summary: str = "") -> None:
+        super().__init__()
+        self.database = database
+        self.summary = summary
+        self._stack = AsyncExitStack()
+        self.repository: AsyncRepository | None = None
+
+    def action_next_theme(self) -> None:
+        from stage.tui.safe import told
+        from stage.tui.state import store_theme
+
+        names = sorted(self.available_themes)
+        index = names.index(self.theme) if self.theme in names else -1
+        self.theme = names[(index + 1) % len(names)]
+        store_theme(self.theme)
+        told(self, f"Theme is now {self.theme}")
+
+    async def on_mount(self) -> None:
+        from stage.storage import open_repository
+        from stage.tui.state import load_theme
+
+        chosen = load_theme()
+        if chosen in self.available_themes:
+            self.theme = chosen
+        self.repository = await self._stack.enter_async_context(open_repository(self.database))
+        self.push_screen(PostingsScreen())
+        if self.summary:
+            self.push_screen(SplashScreen(self.summary))
+
+    async def on_unmount(self) -> None:
+        await self._stack.aclose()
+
+
+async def summarize(database: Path) -> str:
+    import sqlite3
+
+    from stage.companies import RegistryError, load_companies
+    from stage.domain import JobFilters
+    from stage.storage import open_repository
+
+    try:
+        employers = len(load_companies(None))
+    except RegistryError:
+        employers = 0
+    try:
+        async with open_repository(database) as repo:
+            postings = await repo.count_jobs(JobFilters(limit=1))
+            last = await repo.last_sync_at()
+    except (OSError, sqlite3.DatabaseError):
+        return "no database yet — run stage sync"
+    when = f"last sync {last.astimezone().strftime('%Y-%m-%d %H:%M')}" if last else "never synced"
+    return f"{postings:,} postings · {employers:,} employers · {when}"
+
+
+def launch(database: Path, summary: str = "") -> None:
+    StageApp(database, summary).run()

stage/tui/help.py

@@ -0,0 +1,26 @@
+from textual.widgets import Static
+
+
+class HelpOverlay:
+    HELP_TEXT = ""
+
+    def dismiss_overlays(self) -> None:
+        return
+
+    def action_help(self) -> None:
+        panel = self.query_one("#help", Static)  # type: ignore[attr-defined]
+        showing = not panel.has_class("visible")
+        if showing:
+            self.dismiss_overlays()
+            panel.update(self.HELP_TEXT)
+        panel.set_class(showing, "visible")
+
+    def close_help(self) -> bool:
+        panel = self.query_one("#help", Static)  # type: ignore[attr-defined]
+        if panel.has_class("visible"):
+            panel.remove_class("visible")
+            return True
+        return False
+
+
+__all__ = ["HelpOverlay"]

stage/tui/safe.py

@@ -0,0 +1,21 @@
+from rich.markup import escape
+from rich.text import Text
+
+from stage.domain.text import sanitize
+
+
+def cell(value: str, width: int = 0) -> Text:
+    clean = sanitize(value)
+    return Text(clean[:width] if width else clean, no_wrap=True)
+
+
+def quoted(value: str, width: int = 0) -> str:
+    clean = sanitize(value)
+    return escape(clean[:width] if width else clean)
+
+
+def told(screen: object, message: str, severity: str = "information") -> None:
+    screen.notify(sanitize(message), severity=severity, markup=False)  # type: ignore[attr-defined]
+
+
+__all__ = ["cell", "quoted", "told"]

stage/tui/screens/boards.py

@@ -0,0 +1,186 @@
+from typing import TYPE_CHECKING
+
+from textual import work
+from textual.app import ComposeResult
+from textual.binding import Binding
+from textual.screen import Screen
+from textual.widgets import DataTable, Footer, Header, Static
+
+from stage.domain.text import sanitize
+from stage.tui.help import HelpOverlay
+from stage.tui.safe import cell, quoted, told
+
+if TYPE_CHECKING:
+    from stage.storage import AsyncRepository
+
+COLUMNS = ("State", "Company", "Platform", "Postings", "Last success")
+
+
+HELP_TEXT = """[b]Board health[/b]
+  up down  move between employers
+  e        enable this employer
+  d        disable it, twice to confirm
+  r        reload from the database
+
+[dim]? closes this   escape goes back[/dim]"""
+
+
+class BoardsScreen(HelpOverlay, Screen[None]):
+    HELP_TEXT = HELP_TEXT
+    BINDINGS = [
+        Binding("e", "enable", "enable"),
+        Binding("d", "disable", "disable"),
+        Binding("question_mark", "help", "keys"),
+        Binding("escape", "back", "back"),
+        Binding("r", "reload", "reload", show=False),
+    ]
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._companies: tuple[str, ...] = ()
+        self._arming_disable: str | None = None
+
+    @property
+    def repository(self) -> "AsyncRepository | None":
+        from stage.tui.app import StageApp
+
+        app = self.app
+        return app.repository if isinstance(app, StageApp) else None
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield Static("", id="chips")
+        yield DataTable(id="results", cursor_type="row")
+        yield Static("", id="detail")
+        yield Static("", id="help")
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self._mark_loading()
+        self.query_one("#results", DataTable).add_columns(*COLUMNS)
+        self.load()
+
+    def _mark_loading(self) -> None:
+        self.query_one("#chips", Static).update("[dim]loading…[/dim]")
+
+    def on_data_table_row_highlighted(self) -> None:
+        self._arming_disable = None
+
+    def selected_company(self) -> str | None:
+        table = self.query_one("#results", DataTable)
+        row = table.cursor_row
+        if not self._companies or row < 0 or row >= len(self._companies):
+            return None
+        return self._companies[row]
+
+    @work(exclusive=True)
+    async def load(self) -> None:
+        from stage.companies import RegistryError, load_companies
+        from stage.services.coverage import coverage
+
+        repo = self.repository
+        if repo is None:
+            return
+        try:
+            companies = load_companies(None)
+        except RegistryError as exc:
+            self.query_one("#detail", Static).update(f"[red]{quoted(str(exc))}[/red]")
+            return
+        report = await coverage(repo, companies)
+        table = self.query_one("#results", DataTable)
+        table.clear()
+        names = []
+        for row in report.rows:
+            seen = (
+                row.last_success_at.astimezone().strftime("%Y-%m-%d")
+                if row.last_success_at
+                else "—"
+            )
+            table.add_row(
+                cell(row.state.value),
+                cell(row.company, 26),
+                cell(row.platform),
+                cell(str(row.postings)),
+                cell(seen),
+            )
+            names.append(row.company)
+        listed = set(names)
+        for company in companies:
+            if company.enabled or company.name in listed:
+                continue
+            table.add_row(
+                cell("disabled"),
+                cell(company.name, 26),
+                cell(company.platform.value),
+                cell("0"),
+                cell("-"),
+            )
+            names.append(company.name)
+        self._companies = tuple(names)
+        self.query_one("#chips", Static).update(
+            f"[b]{report.enabled}[/b] enabled · [dim]{report.disabled} disabled · "
+            f"{len(report.gaps)} producing nothing[/dim]"
+        )
+        self.query_one("#detail", Static).update(
+            "[b]e[/b] enable  [b]d[/b] disable, twice to confirm  [b]?[/b] keys\n"
+            "[dim]A board producing nothing is either genuinely empty or a parser has "
+            "drifted. Check with stage canary before disabling.[/dim]"
+        )
+
+    def action_disable(self) -> None:
+        company = self.selected_company()
+        if company is None:
+            told(self, "Highlight a board first.", "warning")
+            return
+        if self._arming_disable != company:
+            self._arming_disable = company
+            told(self, f"Press d again to disable {quoted(company)}.", "warning")
+            return
+        self._arming_disable = None
+        self._set_enabled(False)
+
+    def action_enable(self) -> None:
+        self._arming_disable = None
+        self._set_enabled(True)
+
+    def _set_enabled(self, enabled: bool) -> None:
+        from dataclasses import replace
+
+        from stage.companies import RegistryError, update_registry
+        from stage.domain import Company
+
+        company = self.selected_company()
+        if company is None:
+            told(self, "Highlight a board first.", "warning")
+            return
+
+        def apply(rows: tuple[Company, ...]) -> tuple[list[Company], int]:
+            changed = 0
+            updated = []
+            for row in rows:
+                if row.name == company and row.enabled != enabled:
+                    updated.append(replace(row, enabled=enabled))
+                    changed += 1
+                    continue
+                updated.append(row)
+            return updated, changed
+
+        try:
+            _, changed = update_registry(apply)
+        except (RegistryError, OSError) as exc:
+            told(self, f"Could not write the registry: {sanitize(str(exc))}", "error")
+            return
+        if not changed:
+            state = "enabled" if enabled else "disabled"
+            told(self, f"{sanitize(company)} is already {state}.", "warning")
+            return
+        told(self, f"{'Enabled' if enabled else 'Disabled'} {sanitize(company)}")
+        self.load()
+
+    def action_reload(self) -> None:
+        self.load()
+
+    def action_back(self) -> None:
+        if self.close_help():
+            return
+        self.dismiss(None)