stage-cli 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. stage/__init__.py +1 -0
  2. stage/__main__.py +8 -0
  3. stage/banner.py +32 -0
  4. stage/bootstrap/__init__.py +0 -0
  5. stage/bootstrap/openjobs.py +392 -0
  6. stage/classify/__init__.py +29 -0
  7. stage/classify/eligibility.py +115 -0
  8. stage/classify/internship.py +64 -0
  9. stage/classify/role.py +91 -0
  10. stage/classify/scope.py +47 -0
  11. stage/cli/__init__.py +0 -0
  12. stage/cli/app.py +4 -0
  13. stage/cli/commands/__init__.py +8 -0
  14. stage/cli/commands/discovery.py +294 -0
  15. stage/cli/commands/insight.py +494 -0
  16. stage/cli/commands/pipeline.py +337 -0
  17. stage/cli/commands/postings.py +473 -0
  18. stage/cli/commands/schedule.py +171 -0
  19. stage/cli/housekeeping.py +64 -0
  20. stage/cli/logfile.py +56 -0
  21. stage/cli/notify.py +170 -0
  22. stage/cli/options.py +678 -0
  23. stage/cli/render.py +1398 -0
  24. stage/cli/runlock.py +74 -0
  25. stage/cli/schedule.py +702 -0
  26. stage/cli/schedule_state.py +363 -0
  27. stage/cli/selection.py +83 -0
  28. stage/cli/serialize.py +196 -0
  29. stage/companies.py +542 -0
  30. stage/data/companies/a.yaml +1289 -0
  31. stage/data/companies/b.yaml +900 -0
  32. stage/data/companies/c.yaml +1377 -0
  33. stage/data/companies/d.yaml +497 -0
  34. stage/data/companies/e.yaml +519 -0
  35. stage/data/companies/f.yaml +454 -0
  36. stage/data/companies/g.yaml +601 -0
  37. stage/data/companies/h.yaml +446 -0
  38. stage/data/companies/i.yaml +503 -0
  39. stage/data/companies/j.yaml +138 -0
  40. stage/data/companies/k.yaml +278 -0
  41. stage/data/companies/l.yaml +402 -0
  42. stage/data/companies/m.yaml +937 -0
  43. stage/data/companies/n.yaml +549 -0
  44. stage/data/companies/o.yaml +371 -0
  45. stage/data/companies/other.yaml +58 -0
  46. stage/data/companies/p.yaml +825 -0
  47. stage/data/companies/q.yaml +121 -0
  48. stage/data/companies/r.yaml +583 -0
  49. stage/data/companies/s.yaml +1140 -0
  50. stage/data/companies/t.yaml +817 -0
  51. stage/data/companies/u.yaml +196 -0
  52. stage/data/companies/v.yaml +325 -0
  53. stage/data/companies/w.yaml +353 -0
  54. stage/data/companies/x.yaml +67 -0
  55. stage/data/companies/y.yaml +36 -0
  56. stage/data/companies/z.yaml +146 -0
  57. stage/data/fonts/DejaVuSans.LICENSE.txt +99 -0
  58. stage/data/fonts/DejaVuSans.ttf +0 -0
  59. stage/data/lexicon/company_tokens.yaml +228 -0
  60. stage/data/lexicon/eligibility.yaml +455 -0
  61. stage/data/lexicon/inclusive_suffixes.yaml +37 -0
  62. stage/data/lexicon/internship.yaml +187 -0
  63. stage/data/lexicon/language.yaml +226 -0
  64. stage/data/lexicon/locations.yaml +1159 -0
  65. stage/data/lexicon/roles.yaml +2012 -0
  66. stage/data/lexicon/terms.yaml +76 -0
  67. stage/data/lexicon/workday_facets.yaml +27 -0
  68. stage/data/seed_companies.yaml +198 -0
  69. stage/dedup/__init__.py +19 -0
  70. stage/dedup/identity.py +113 -0
  71. stage/dedup/resolve.py +97 -0
  72. stage/domain/__init__.py +244 -0
  73. stage/domain/company.py +49 -0
  74. stage/domain/coverage.py +86 -0
  75. stage/domain/custom_board.py +92 -0
  76. stage/domain/discovery.py +94 -0
  77. stage/domain/enums.py +114 -0
  78. stage/domain/events.py +204 -0
  79. stage/domain/filters.py +27 -0
  80. stage/domain/health.py +169 -0
  81. stage/domain/ids.py +48 -0
  82. stage/domain/job.py +47 -0
  83. stage/domain/matching.py +15 -0
  84. stage/domain/priority.py +34 -0
  85. stage/domain/quarantine.py +39 -0
  86. stage/domain/rate_state.py +78 -0
  87. stage/domain/retention.py +20 -0
  88. stage/domain/rotation.py +46 -0
  89. stage/domain/signals.py +12 -0
  90. stage/domain/sync_run.py +35 -0
  91. stage/domain/text.py +113 -0
  92. stage/domain/validator.py +14 -0
  93. stage/domain/visits.py +60 -0
  94. stage/domain/workday.py +38 -0
  95. stage/http/__init__.py +58 -0
  96. stage/http/breaker.py +53 -0
  97. stage/http/cache.py +44 -0
  98. stage/http/client.py +725 -0
  99. stage/http/profiles.py +101 -0
  100. stage/lexicon.py +370 -0
  101. stage/normalize/__init__.py +16 -0
  102. stage/normalize/language.py +47 -0
  103. stage/normalize/location.py +271 -0
  104. stage/normalize/terms.py +153 -0
  105. stage/normalize/urls.py +122 -0
  106. stage/paths.py +86 -0
  107. stage/py.typed +0 -0
  108. stage/services/__init__.py +0 -0
  109. stage/services/canary.py +120 -0
  110. stage/services/coverage.py +231 -0
  111. stage/services/discover.py +747 -0
  112. stage/services/export.py +274 -0
  113. stage/services/health.py +237 -0
  114. stage/services/maintenance.py +225 -0
  115. stage/services/quarantine.py +20 -0
  116. stage/services/query.py +86 -0
  117. stage/services/sync.py +1257 -0
  118. stage/sources/__init__.py +82 -0
  119. stage/sources/_text.py +79 -0
  120. stage/sources/ashby.py +93 -0
  121. stage/sources/bamboohr.py +80 -0
  122. stage/sources/base.py +225 -0
  123. stage/sources/breezy.py +90 -0
  124. stage/sources/collage.py +60 -0
  125. stage/sources/community_feeds.py +142 -0
  126. stage/sources/curated_markdown.py +289 -0
  127. stage/sources/custom_json.py +610 -0
  128. stage/sources/espresso.py +154 -0
  129. stage/sources/feed.py +44 -0
  130. stage/sources/greenhouse.py +104 -0
  131. stage/sources/jobbank.py +147 -0
  132. stage/sources/jobvite.py +133 -0
  133. stage/sources/lever.py +76 -0
  134. stage/sources/oracle_cloud.py +187 -0
  135. stage/sources/platforms.py +609 -0
  136. stage/sources/quebec_emploi.py +146 -0
  137. stage/sources/recruitee.py +96 -0
  138. stage/sources/simplify.py +110 -0
  139. stage/sources/smartrecruiters.py +216 -0
  140. stage/sources/speedyapply.py +200 -0
  141. stage/sources/themuse.py +157 -0
  142. stage/sources/workable.py +83 -0
  143. stage/sources/workday.py +524 -0
  144. stage/sources/zshah.py +99 -0
  145. stage/storage/__init__.py +29 -0
  146. stage/storage/migrations/0001_initial.sql +239 -0
  147. stage/storage/migrations/__init__.py +135 -0
  148. stage/storage/repository.py +213 -0
  149. stage/storage/search.py +28 -0
  150. stage/storage/sqlite_repo.py +1586 -0
  151. stage/storage/writer.py +249 -0
  152. stage/tui/__init__.py +0 -0
  153. stage/tui/app.py +82 -0
  154. stage/tui/help.py +26 -0
  155. stage/tui/safe.py +21 -0
  156. stage/tui/screens/__init__.py +0 -0
  157. stage/tui/screens/boards.py +186 -0
  158. stage/tui/screens/postings.py +509 -0
  159. stage/tui/screens/review.py +209 -0
  160. stage/tui/screens/splash.py +37 -0
  161. stage/tui/screens/stats.py +124 -0
  162. stage/tui/screens/sync.py +194 -0
  163. stage/tui/state.py +160 -0
  164. stage/tui/theme.tcss +205 -0
  165. stage/tui/widgets/__init__.py +0 -0
  166. stage_cli-1.0.0.dist-info/METADATA +379 -0
  167. stage_cli-1.0.0.dist-info/RECORD +170 -0
  168. stage_cli-1.0.0.dist-info/WHEEL +4 -0
  169. stage_cli-1.0.0.dist-info/entry_points.txt +2 -0
  170. stage_cli-1.0.0.dist-info/licenses/LICENSE +21 -0
stage/cli/render.py ADDED
@@ -0,0 +1,1398 @@
1
+ import json
2
+ from collections.abc import AsyncIterator, Callable, Sequence
3
+ from datetime import UTC, date, datetime
4
+ from typing import TYPE_CHECKING, TextIO
5
+
6
+ from rich.console import Console
7
+ from rich.markup import escape
8
+ from rich.table import Table
9
+ from rich.text import Text
10
+
11
+ if TYPE_CHECKING:
12
+ from stage.domain import WorkdayCrawl
13
+ from stage.services.canary import CanaryReport
14
+ from stage.services.coverage import CoverageReport
15
+ from stage.services.export import ExportResult
16
+ from stage.services.health import DoctorReport, SourceHealth, StatsReport
17
+ from stage.services.query import JobListing, PostingDetail
18
+
19
+ from stage.domain import (
20
+ UNKNOWN_TERM,
21
+ BucketPlan,
22
+ CandidateSkipped,
23
+ CompanyDeferred,
24
+ CompanyFailed,
25
+ CompanyFinished,
26
+ CompanyUnchanged,
27
+ CoverageState,
28
+ DiscoveryEvent,
29
+ DiscoveryFinished,
30
+ DiscoveryStarted,
31
+ IntegrityRepair,
32
+ Job,
33
+ PlannedRequest,
34
+ PlatformCandidate,
35
+ PlatformProbed,
36
+ ProbeVerdict,
37
+ QuarantinedJob,
38
+ RateState,
39
+ RequestLogged,
40
+ SourceBlocked,
41
+ SourceCapped,
42
+ SourceFailed,
43
+ SourceFinished,
44
+ SourceFresh,
45
+ SourceResting,
46
+ SourceRotated,
47
+ SourceStarted,
48
+ SyncEvent,
49
+ SyncFinished,
50
+ SyncOutcome,
51
+ SyncStarted,
52
+ UnroutableCompanies,
53
+ UrlResolved,
54
+ UrlUnrecognized,
55
+ VisitState,
56
+ VolumeVerdict,
57
+ web_url,
58
+ )
59
+ from stage.domain.text import first_line as _first_line
60
+ from stage.domain.text import sanitize as _sanitize
61
+ from stage.domain.text import summary as _summary
62
+ from stage.domain.text import truncate as _truncate
63
+
64
+
65
+ def terminal() -> Console:
66
+ return Console(emoji=False)
67
+
68
+
69
+ def splash(console: Console) -> None:
70
+ from stage.banner import ACCENT, banner
71
+
72
+ console.print(Text(banner(console.width), style=f"bold {ACCENT}"))
73
+
74
+
75
+ def sanitize(value: str) -> str:
76
+ return escape(_sanitize(value))
77
+
78
+
79
+ def truncate(value: str, width: int) -> str:
80
+ return escape(_truncate(value, width))
81
+
82
+
83
+ def first_line(value: str) -> str:
84
+ return escape(_first_line(value))
85
+
86
+
87
+ def summary(value: str, width: int) -> str:
88
+ return escape(_summary(value, width))
89
+
90
+
91
+ def plain(value: str, style: str = "") -> Text:
92
+ return Text(_sanitize(value), style=style)
93
+
94
+
95
+ def clipped(value: str, width: int, style: str = "") -> Text:
96
+ return Text(_truncate(value, width), style=style)
97
+
98
+
99
+ def failure(exc: BaseException) -> Text:
100
+ return Text(_sanitize(str(exc)), style="red")
101
+
102
+
103
+ def place(raw: str) -> str:
104
+ from stage.normalize.location import display_location
105
+
106
+ return display_location(raw)
107
+
108
+
109
+ def rule() -> Text:
110
+ return Text("-" * 60, style="dim")
111
+
112
+
113
+ def quoted(value: str, width: int) -> str:
114
+ return f"'{truncate(value, width)}'"
115
+
116
+
117
+ def _link(text: Text, raw: str) -> None:
118
+ url = web_url(raw)
119
+ if url is not None:
120
+ text.stylize(f"link {url}")
121
+
122
+
123
+ def _duration(seconds: float) -> str:
124
+ if seconds >= 3600:
125
+ return f"{seconds / 3600:.1f}h"
126
+ if seconds >= 60:
127
+ return f"{seconds / 60:.0f}m"
128
+ return f"{seconds:.0f}s"
129
+
130
+
131
+ def render_rate_state(
132
+ console: Console,
133
+ states: Sequence[RateState],
134
+ now: datetime,
135
+ *,
136
+ verbose: bool = False,
137
+ ) -> None:
138
+ if not states:
139
+ console.print("[dim]No rate state stored. Nothing has been throttled or blocked.[/dim]")
140
+ return
141
+
142
+ table = Table(box=None, pad_edge=False, header_style="bold")
143
+ table.add_column("bucket")
144
+ table.add_column("status")
145
+ table.add_column("interval", justify="right")
146
+ table.add_column("fails", justify="right")
147
+ table.add_column("rotation")
148
+ table.add_column("reason")
149
+
150
+ for state in sorted(states, key=lambda item: item.bucket):
151
+ if state.is_blocked(now):
152
+ status = f"[red]blocked {_duration(state.blocks_remaining_s(now))}[/red]"
153
+ elif state.blocked_until is not None:
154
+ status = "[yellow]block expired[/yellow]"
155
+ elif state.min_interval_override is not None:
156
+ status = "[yellow]tightened[/yellow]"
157
+ else:
158
+ status = "[green]clear[/green]"
159
+ override = (
160
+ f"{state.min_interval_override:.2f}s"
161
+ if state.min_interval_override is not None
162
+ else "[dim]default[/dim]"
163
+ )
164
+ table.add_row(
165
+ truncate(state.bucket, 34),
166
+ status,
167
+ override,
168
+ str(state.consecutive_failures),
169
+ truncate(state.rotation_cursor, 24) or "[dim]—[/dim]",
170
+ summary(state.reason, 4000 if verbose else 40) or "[dim]—[/dim]",
171
+ )
172
+
173
+ console.print(table)
174
+ console.print(
175
+ "[dim]Tightening decays on each clean run. "
176
+ "[bold]stage sources --reset-rate-limit <bucket>[/bold] resets a block now.[/dim]"
177
+ )
178
+
179
+
180
+ NARROW_COLUMNS = 70
181
+ CACHE_RATIO_MINIMUM = 20
182
+
183
+
184
+ def _age_style(first_seen: datetime, now: datetime) -> tuple[str, str]:
185
+ age_days = (now - first_seen).days
186
+ if age_days <= 1:
187
+ return "bold green", "new"
188
+ if age_days <= 3:
189
+ return "green", "recent"
190
+ if age_days <= 7:
191
+ return "default", "week"
192
+ return "dim", "older"
193
+
194
+
195
+ def render_jobs(
196
+ console: Console,
197
+ jobs: Sequence[Job],
198
+ *,
199
+ total_matching: int,
200
+ window_days: int | None,
201
+ last_sync_at: datetime | None,
202
+ now: datetime | None = None,
203
+ numbered: int | None = None,
204
+ hint: str = "",
205
+ ) -> None:
206
+ moment = now or datetime.now(UTC)
207
+ if not jobs:
208
+ console.print(_empty_state(window_days, last_sync_at, moment, hint))
209
+ return
210
+ addressable = len(jobs) if numbered is None else numbered
211
+
212
+ narrow = console.width < NARROW_COLUMNS
213
+ row_width = max(2, len(str(len(jobs))))
214
+ age_width = 6
215
+ seen_width = 0 if narrow else 10
216
+ company_width = 14 if narrow else 18
217
+ location_width = 0 if narrow else 18
218
+ used = row_width + age_width + seen_width + company_width + location_width
219
+ columns = 4 if narrow else 6
220
+ title_width = max(16, console.width - used - 2 * columns)
221
+
222
+ table = Table(box=None, pad_edge=False, header_style="bold")
223
+ table.add_column("#", width=row_width, justify="right", no_wrap=True)
224
+ table.add_column("Age", width=age_width, no_wrap=True)
225
+ if not narrow:
226
+ table.add_column("Seen", width=seen_width, no_wrap=True)
227
+ table.add_column("Company", width=company_width, no_wrap=True, overflow="ellipsis")
228
+ table.add_column("Title", width=title_width, no_wrap=True, overflow="ellipsis")
229
+ if not narrow:
230
+ table.add_column("Location", width=location_width, no_wrap=True, overflow="ellipsis")
231
+
232
+ for position, job in enumerate(jobs, start=1):
233
+ style, label = _age_style(job.first_seen, moment)
234
+ title = clipped(job.title_raw, title_width, style=style)
235
+ _link(title, job.apply_url_raw)
236
+ number = str(position) if position <= addressable else ""
237
+ cells = [Text(number, style="dim"), Text(label, style=style)]
238
+ if not narrow:
239
+ cells.append(Text(job.first_seen.astimezone().strftime("%Y-%m-%d")))
240
+ cells.append(Text(truncate(job.company, 22)))
241
+ cells.append(title)
242
+ if not narrow:
243
+ cells.append(Text(truncate(place(job.location_raw) or "—", 22)))
244
+ table.add_row(*cells)
245
+
246
+ console.print(table)
247
+ shown = len(jobs)
248
+ suffix = f" of {total_matching}" if total_matching > shown else ""
249
+ capped = f" Only the first {addressable} are numbered." if addressable < shown else ""
250
+ console.print(
251
+ f"\n[dim]{shown} posting(s){suffix}. "
252
+ f"stage show 1 or stage open 1 acts on a numbered row.{capped}[/dim]"
253
+ )
254
+
255
+
256
+ def _dropped_punctuation(query: str) -> str:
257
+ import unicodedata
258
+
259
+ def carries_punctuation(word: str) -> bool:
260
+ return any(
261
+ unicodedata.category(character)[0] in "PS" and character != "_" for character in word
262
+ )
263
+
264
+ return ", ".join(word for word in query.split() if carries_punctuation(word))
265
+
266
+
267
+ def render_search(
268
+ console: Console,
269
+ listing: "JobListing",
270
+ *,
271
+ now: datetime | None = None,
272
+ numbered: int | None = None,
273
+ hint: str = "",
274
+ ) -> None:
275
+ if not listing.terms:
276
+ console.print(
277
+ f"[yellow]Nothing searchable in {quoted(listing.query, 40)}.[/yellow] "
278
+ "Search matches whole words. Letters and digits, accents optional."
279
+ )
280
+ return
281
+ if not listing.jobs:
282
+ matched = " ".join(listing.terms)
283
+ why = hint or (
284
+ "Terms are combined with AND and matched as prefixes, so drop a word "
285
+ "to widen it, or relax a filter."
286
+ )
287
+ console.print(f"[yellow]No posting matches {matched!r}.[/yellow] {why}")
288
+ return
289
+ render_jobs(
290
+ console,
291
+ listing.jobs,
292
+ total_matching=listing.total_matching,
293
+ window_days=listing.window_days,
294
+ last_sync_at=listing.last_sync_at,
295
+ now=now,
296
+ numbered=numbered,
297
+ hint=hint,
298
+ )
299
+ console.print(f"[dim]Matched {' '.join(listing.terms)} as prefixes, ranked by relevance.[/dim]")
300
+ dropped = _dropped_punctuation(listing.query)
301
+ if dropped:
302
+ console.print(
303
+ f"[yellow]Punctuation is ignored, so {sanitize(dropped)} was searched as "
304
+ f"{' '.join(listing.terms)}.[/yellow]"
305
+ )
306
+
307
+
308
+ def _field(label: str, value: str) -> Text:
309
+ return Text.assemble((f"{label:<14}", "bold"), value)
310
+
311
+
312
+ def render_posting(console: Console, detail: "PostingDetail", now: datetime | None = None) -> None:
313
+ job = detail.job
314
+ moment = now or datetime.now(UTC)
315
+ style, age = _age_style(job.first_seen, moment)
316
+
317
+ title = plain(job.title_raw, style=f"bold {style}")
318
+ _link(title, job.apply_url_raw)
319
+ console.print(title)
320
+ console.print(plain(job.company, style="cyan"))
321
+ console.print()
322
+
323
+ remote = f" ({job.remote_scope.value})" if job.remote_scope else ""
324
+ where = f"{_sanitize(place(job.location_raw)) or '—'} [{job.location.value}]{remote}"
325
+
326
+ def state(label: str, value: str) -> None:
327
+ if value and value != UNKNOWN_TERM:
328
+ console.print(_field(label, value))
329
+
330
+ console.print(_field("status", job.status.value))
331
+ console.print(_field("location", where))
332
+ state("term", job.term)
333
+ state("role", job.role.value)
334
+ state("language", job.language.value)
335
+ state("degree", job.degree_requirement.value)
336
+ if job.work_auth_flag:
337
+ console.print(_field("work auth", "restricted: the posting states an eligibility limit"))
338
+ if job.compensation:
339
+ console.print(_field("compensation", _sanitize(job.compensation)))
340
+ console.print(_field("first seen", f"{job.first_seen.astimezone():%Y-%m-%d} ({age})"))
341
+ console.print(_field("last seen", f"{job.last_seen.astimezone():%Y-%m-%d}"))
342
+ if job.source_posted_at:
343
+ console.print(_field("source date", f"{job.source_posted_at.astimezone():%Y-%m-%d}"))
344
+ console.print(_field("source", f"{job.source} / {job.board_key}"))
345
+ console.print()
346
+ console.print(plain(_sanitize(job.apply_url_raw) or "—", style="blue"))
347
+ console.print(plain(job.id, style="dim"))
348
+
349
+ if detail.canonical is not None:
350
+ console.print()
351
+ console.print(
352
+ f"[yellow]Linked as a duplicate of[/yellow] {detail.canonical.id} "
353
+ f"({detail.canonical.source}) — that row is the one [bold]stage list[/bold] shows."
354
+ )
355
+ if detail.duplicates:
356
+ console.print()
357
+ console.print(f"[bold]Also published as[/bold] ({len(detail.duplicates)})")
358
+ for other in detail.duplicates:
359
+ console.print(f" {other.source:<16} {truncate(other.title_raw, 48):<48} {other.id}")
360
+
361
+ console.print()
362
+ if job.description.strip():
363
+ console.print("[bold]Description[/bold]")
364
+ console.print(plain(job.description.strip()))
365
+ else:
366
+ console.print(
367
+ "[dim]No description stored. Feeds publish none, and some boards only carry "
368
+ "one on the posting page.[/dim]"
369
+ )
370
+
371
+
372
+ def render_export(console: Console, result: "ExportResult") -> None:
373
+ console.print(export_summary(result))
374
+ for note in result.notes:
375
+ console.print(f" [yellow]{sanitize(note)}[/yellow]")
376
+ if result.notes:
377
+ console.print(
378
+ " [dim]Those characters are absent from the embedded font and were dropped from "
379
+ "the PDF only. Export json or csv to keep them.[/dim]"
380
+ )
381
+
382
+
383
+ def export_summary(result: "ExportResult") -> str:
384
+ truncated = (
385
+ f" [yellow]{result.total_matching - result.count} more match the filters — raise "
386
+ "--limit to include them.[/yellow]"
387
+ if result.total_matching > result.count
388
+ else ""
389
+ )
390
+ return (
391
+ f"Exported {result.count} posting(s) as {result.fmt.value} to "
392
+ f"[bold]{sanitize(str(result.path))}[/bold].{truncated}"
393
+ )
394
+
395
+
396
+ _COVERAGE_STYLE = {
397
+ CoverageState.PRODUCING: "green",
398
+ CoverageState.EMPTY: "yellow",
399
+ CoverageState.FAILING: "red",
400
+ CoverageState.STALE: "yellow",
401
+ CoverageState.NEVER_REACHED: "dim",
402
+ CoverageState.UNROUTABLE: "red",
403
+ }
404
+
405
+
406
+ def render_contradictions(console: Console, report: "CoverageReport") -> None:
407
+ if not report.contradictions:
408
+ console.print("[dim]No review verdict contradicts the registry or has aged out.[/dim]")
409
+ return
410
+ table = Table(
411
+ title="Review verdicts to re-derive", box=None, pad_edge=False, header_style="bold"
412
+ )
413
+ table.add_column("Company")
414
+ table.add_column("Verdict")
415
+ table.add_column("Why it no longer holds")
416
+ for record, reason in report.contradictions:
417
+ table.add_row(
418
+ sanitize(record.company), sanitize(record.disposition.value), sanitize(reason)
419
+ )
420
+ console.print(table)
421
+
422
+
423
+ ROW_PREVIEW = 30
424
+ NAME_PREVIEW = 8
425
+ BOARD_PREVIEW = 10
426
+
427
+
428
+ def render_coverage(
429
+ console: Console,
430
+ report: "CoverageReport",
431
+ now: datetime,
432
+ *,
433
+ include_classified: bool = False,
434
+ include_contradictions: bool = False,
435
+ limit: int | None = ROW_PREVIEW,
436
+ ) -> None:
437
+ if include_contradictions:
438
+ render_contradictions(console, report)
439
+ return
440
+ counts: dict[CoverageState, int] = {}
441
+ for row in report.rows:
442
+ counts[row.state] = counts.get(row.state, 0) + 1
443
+ breakdown = ", ".join(
444
+ f"[{_COVERAGE_STYLE[state]}]{counts[state]} {state.value}[/{_COVERAGE_STYLE[state]}]"
445
+ for state in CoverageState
446
+ if state in counts
447
+ )
448
+ console.print(f"[bold]{report.enabled}[/bold] enabled row(s): {breakdown or 'none'}")
449
+ console.print(f"[dim]{report.disabled} disabled row(s) are not expected to produce.[/dim]")
450
+
451
+ notes = (
452
+ (CoverageState.FAILING, "have never succeeded", "a fetch problem; see stage doctor"),
453
+ (CoverageState.STALE, "have not succeeded lately", "stale rather than empty"),
454
+ (CoverageState.UNROUTABLE, "have no adapter", "enabled rows nothing will ever fetch"),
455
+ (CoverageState.NEVER_REACHED, "have not been polled yet", "no evidence either way"),
456
+ )
457
+ for state, label, note in notes:
458
+ _render_coverage_note(console, report, state, label, note, limit=limit)
459
+
460
+ gaps = report.gaps
461
+ if gaps:
462
+ console.print()
463
+ console.print(
464
+ f"[bold]No internships open right now[/bold] ({len(gaps)}) "
465
+ "[dim]— these boards answered, they just had nothing matching[/dim]"
466
+ )
467
+ table = Table(box=None, pad_edge=False, header_style="bold")
468
+ table.add_column("company")
469
+ table.add_column("board")
470
+ table.add_column("last success", justify="right")
471
+ shown = gaps if limit is None else gaps[:limit]
472
+ for row in shown:
473
+ table.add_row(
474
+ truncate(row.company, 28), truncate(row.board, 38), _ago(row.last_success_at, now)
475
+ )
476
+ console.print(table)
477
+ if len(gaps) > len(shown):
478
+ console.print(
479
+ f" [dim]… and {len(gaps) - len(shown)} more; --all lists every row[/dim]"
480
+ )
481
+ console.print("[dim]Expected most of the year. Internship postings are seasonal.[/dim]")
482
+
483
+ if report.unregistered:
484
+ console.print()
485
+ console.print(
486
+ f"[bold]Seen in a feed, absent from the registry[/bold] ({len(report.unregistered)})"
487
+ )
488
+ table = Table(box=None, pad_edge=False, header_style="bold")
489
+ table.add_column("company")
490
+ table.add_column("postings", justify="right")
491
+ table.add_column("sources")
492
+ listed = report.unregistered if limit is None else report.unregistered[:limit]
493
+ for unknown in listed:
494
+ table.add_row(
495
+ truncate(unknown.company, 34),
496
+ str(unknown.postings),
497
+ ", ".join(unknown.sources),
498
+ )
499
+ console.print(table)
500
+ if len(report.unregistered) > 30:
501
+ console.print(f" [dim]… and {len(report.unregistered) - 30} more[/dim]")
502
+ console.print(
503
+ '[dim]After researching an employer, record it with [bold]stage classify "Company" '
504
+ '--status feed-only --note "why"[/bold]. To identify a career-board URL without '
505
+ "fetching it, use [bold]stage discover --url URL[/bold].[/dim]"
506
+ )
507
+
508
+ if include_classified:
509
+ console.print()
510
+ if report.classifications:
511
+ console.print(f"[bold]Reviewed feed employers[/bold] ({len(report.classifications)})")
512
+ table = Table(box=None, pad_edge=False, header_style="bold")
513
+ table.add_column("company")
514
+ table.add_column("status")
515
+ table.add_column("checked", justify="right")
516
+ table.add_column("note")
517
+ for entry in report.classifications:
518
+ table.add_row(
519
+ truncate(entry.company, 26),
520
+ entry.disposition.value,
521
+ entry.checked_on.date().isoformat(),
522
+ truncate(entry.note, 56),
523
+ )
524
+ console.print(table)
525
+ else:
526
+ console.print("[dim]No feed employers have been reviewed yet.[/dim]")
527
+
528
+
529
+ def _render_coverage_note(
530
+ console: Console,
531
+ report: "CoverageReport",
532
+ state: CoverageState,
533
+ label: str,
534
+ note: str,
535
+ *,
536
+ limit: int | None = NAME_PREVIEW,
537
+ ) -> None:
538
+ rows = [row for row in report.rows if row.state is state]
539
+ if not rows:
540
+ return
541
+ cap = len(rows) if limit is None else NAME_PREVIEW
542
+ names = ", ".join(truncate(row.company, 24) for row in rows[:cap])
543
+ more = f" and {len(rows) - cap} more" if len(rows) > cap else ""
544
+ console.print()
545
+ console.print(f"[bold]{len(rows)} row(s) {label}[/bold] — {note}")
546
+ console.print(f" [dim]{names}{more}[/dim]")
547
+
548
+
549
+ def _ago(when: datetime | None, now: datetime) -> str:
550
+ if when is None:
551
+ return "[red]never[/red]"
552
+ return _duration((now - when).total_seconds()) + " ago"
553
+
554
+
555
+ def _ratio(value: float | None) -> str:
556
+ return "[dim]—[/dim]" if value is None else f"{value:.0%}"
557
+
558
+
559
+ def render_source_health(
560
+ console: Console, sources: Sequence["SourceHealth"], stale_after_days: int
561
+ ) -> None:
562
+ if not sources:
563
+ console.print("[dim]No sync has run yet. Run stage sync.[/dim]")
564
+ return
565
+
566
+ table = Table(box=None, pad_edge=False, header_style="bold")
567
+ table.add_column("source", no_wrap=True)
568
+ table.add_column("open", justify="right")
569
+ table.add_column("volume", no_wrap=True)
570
+ table.add_column("ok", justify="right")
571
+ table.add_column("cache", justify="right")
572
+ table.add_column("mid", justify="right")
573
+ table.add_column("slow", justify="right")
574
+ table.add_column("calls", justify="right")
575
+ table.add_column("boards")
576
+
577
+ for source in sources:
578
+ verdict = source.volume.verdict
579
+ if verdict is VolumeVerdict.COLLAPSED:
580
+ volume = "[red]collapsed[/red]"
581
+ elif verdict is VolumeVerdict.DROPPED:
582
+ volume = "[red]dropped[/red]"
583
+ elif verdict is VolumeVerdict.UNPROVEN:
584
+ volume = "[dim]unproven[/dim]"
585
+ else:
586
+ volume = "[green]steady[/green]"
587
+
588
+ failing, stale = len(source.failing_boards), len(source.stale_boards)
589
+ if failing:
590
+ boards = f"[red]{failing} failing[/red]"
591
+ if stale:
592
+ boards += f", [yellow]{stale} stale[/yellow]"
593
+ elif stale:
594
+ boards = f"[yellow]{stale} stale[/yellow]"
595
+ elif source.boards:
596
+ boards = f"[green]{len(source.boards)} ok[/green]"
597
+ else:
598
+ boards = "[dim]—[/dim]"
599
+
600
+ rate = source.success_rate
601
+ if rate is None:
602
+ success = "[dim]—[/dim]"
603
+ elif rate < 1.0:
604
+ success = f"[yellow]{rate:.0%}[/yellow]"
605
+ else:
606
+ success = f"[green]{rate:.0%}[/green]"
607
+
608
+ table.add_row(
609
+ source.source,
610
+ str(source.stored),
611
+ volume,
612
+ success,
613
+ _ratio(source.cache_hit_ratio),
614
+ f"{source.latency_p50_ms:.0f}ms",
615
+ f"{source.latency_p95_ms:.0f}ms",
616
+ str(source.requests),
617
+ boards,
618
+ )
619
+
620
+ console.print(table)
621
+ console.print(
622
+ "[dim]open: postings held now. ok: successful fetches. cache: served "
623
+ "unchanged.\nmid and slow: typical and worst response times. calls: requests "
624
+ f"made.\nA board is stale after {stale_after_days} days without a success, and "
625
+ "failing when it has never succeeded.[/dim]"
626
+ )
627
+
628
+
629
+ def render_workday_crawl_progress(console: Console, crawls: Sequence["WorkdayCrawl"]) -> None:
630
+ console.print("[bold]Workday crawl progress[/bold]")
631
+ if not crawls:
632
+ console.print("[dim]No incomplete Workday crawl is retained.[/dim]")
633
+ return
634
+
635
+ table = Table(box=None, pad_edge=False, header_style="bold")
636
+ table.add_column("board")
637
+ table.add_column("next offset", justify="right")
638
+ table.add_column("reported total", justify="right")
639
+ for crawl in crawls:
640
+ table.add_row(
641
+ truncate(crawl.board, 48),
642
+ str(crawl.next_offset),
643
+ str(crawl.total) if crawl.total is not None else "[dim]unknown[/dim]",
644
+ )
645
+ console.print(table)
646
+ console.print(
647
+ "[dim]These boards are part-way through a paged crawl and resume next sync. "
648
+ "Their postings stay open until a full pass finishes.[/dim]"
649
+ )
650
+
651
+
652
+ def render_board_health(
653
+ console: Console,
654
+ sources: Sequence["SourceHealth"],
655
+ now: datetime,
656
+ *,
657
+ verbose: bool = False,
658
+ ) -> None:
659
+ rows = [
660
+ board
661
+ for source in sources
662
+ for board in source.boards
663
+ if board.state is not VisitState.HEALTHY
664
+ ]
665
+ if not rows:
666
+ console.print("[green]Every board that rotation has reached succeeded recently.[/green]")
667
+ return
668
+
669
+ table = Table(box=None, pad_edge=False, header_style="bold")
670
+ table.add_column("board")
671
+ table.add_column("source")
672
+ table.add_column("state")
673
+ table.add_column("last success", justify="right")
674
+ table.add_column("fails", justify="right")
675
+ table.add_column("error")
676
+
677
+ for board in rows:
678
+ colour = "red" if board.state is VisitState.FAILING else "yellow"
679
+ table.add_row(
680
+ truncate(board.label, 30),
681
+ board.source,
682
+ f"[{colour}]{board.state.value}[/{colour}]",
683
+ _ago(board.last_success_at, now),
684
+ str(board.consecutive_failures),
685
+ summary(board.last_error, 4000 if verbose else 40) or "[dim]—[/dim]",
686
+ )
687
+
688
+ console.print(table)
689
+ hint = (
690
+ ""
691
+ if verbose or not any(len(board.last_error or "") > 40 for board in rows)
692
+ else " --verbose prints each error in full."
693
+ )
694
+ console.print(f"[dim]Boards not listed have not been reached yet.{hint}[/dim]")
695
+
696
+
697
+ def render_canary(
698
+ console: Console,
699
+ report: "CanaryReport",
700
+ *,
701
+ verbose: bool = False,
702
+ ) -> None:
703
+ table = Table(box=None, pad_edge=False, header_style="bold")
704
+ table.add_column("source")
705
+ table.add_column("board")
706
+ table.add_column("result")
707
+ table.add_column("postings", justify="right")
708
+ table.add_column("note")
709
+
710
+ for probe in report.probes:
711
+ if probe.is_failure:
712
+ result = "[red]failed[/red]"
713
+ elif probe.is_unreachable:
714
+ result = "[yellow]unreachable[/yellow]"
715
+ elif probe.is_empty:
716
+ result = "[red]no postings[/red]"
717
+ elif probe.unchanged:
718
+ result = "[dim]unchanged[/dim]"
719
+ else:
720
+ result = "[green]ok[/green]"
721
+ note = probe.error or probe.degraded
722
+ table.add_row(
723
+ probe.source,
724
+ truncate(probe.company, 28),
725
+ result,
726
+ "[dim]—[/dim]" if probe.unchanged else str(probe.fetched),
727
+ summary(note, 4000 if verbose else 44) or "[dim]—[/dim]",
728
+ )
729
+
730
+ console.print(table)
731
+ if report.skipped_platforms:
732
+ console.print(
733
+ f"[dim]Skipped {', '.join(report.skipped_platforms)} : bot-protected, so never "
734
+ "probed on a schedule.[/dim]"
735
+ )
736
+ console.print()
737
+ if report.unreachable:
738
+ console.print(
739
+ f"[yellow]{len(report.unreachable)} board(s) refused or dropped the request.[/yellow] "
740
+ "That is their server, not the parser. [bold]stage doctor[/bold] tracks "
741
+ "repeat failures."
742
+ )
743
+ if report.passed:
744
+ console.print(
745
+ f"[green]{len(report.probes) - len(report.unreachable)} board(s) still answer "
746
+ "the shape we parse.[/green]"
747
+ )
748
+ else:
749
+ console.print(
750
+ f"[red]{len(report.failures)} failed, {len(report.empties)} returned "
751
+ "nothing.[/red] Rebuild the fixture from the captured payload."
752
+ )
753
+
754
+
755
+ def render_repairs(console: "Console", repairs: Sequence[IntegrityRepair]) -> None:
756
+ if not repairs:
757
+ return
758
+ console.print("[bold]Repaired[/bold]")
759
+ for entry in repairs:
760
+ console.print(f" {entry.repaired} × {sanitize(entry.check)} — {sanitize(entry.detail)}")
761
+ console.print()
762
+
763
+
764
+ def render_doctor(
765
+ console: Console,
766
+ report: "DoctorReport",
767
+ now: datetime,
768
+ *,
769
+ limit: int | None = BOARD_PREVIEW,
770
+ verbose: bool = False,
771
+ ) -> None:
772
+ console.print(f"[bold]schema[/bold] v{report.schema_version}")
773
+ if report.never_synced:
774
+ console.print(
775
+ "[yellow]No sync has ever run here[/yellow], so integrity is clean by "
776
+ "default. Run stage sync."
777
+ )
778
+ else:
779
+ console.print(f"[bold]last sync[/bold] {_ago(report.last_sync_at, now)}")
780
+ console.print()
781
+
782
+ problems = report.integrity_problems
783
+ if problems:
784
+ console.print("[bold red]Integrity[/bold red]")
785
+ for finding in problems:
786
+ console.print(f" [red]{finding.count}[/red] {finding.check} — {finding.detail}")
787
+ else:
788
+ console.print(
789
+ f"[bold green]Integrity[/bold green] all {len(report.integrity)} checks clean"
790
+ )
791
+ console.print()
792
+
793
+ console.print("[bold]Sources[/bold]")
794
+ render_source_health(console, report.sources, report.stale_after_days)
795
+ console.print()
796
+ render_workday_crawl_progress(console, report.workday_crawls)
797
+
798
+ alerts = report.volume_alerts
799
+ if alerts:
800
+ console.print()
801
+ console.print("[bold red]Volume[/bold red]")
802
+ for source in alerts:
803
+ console.print(f" [red]{source.source}[/red] {source.volume.detail}")
804
+
805
+ if report.blocks:
806
+ console.print()
807
+ console.print("[bold red]Blocked buckets[/bold red]")
808
+ for state in report.blocks:
809
+ console.print(
810
+ f" [red]{state.bucket}[/red] for another "
811
+ f"{_duration(state.blocks_remaining_s(now))} — "
812
+ f"{first_line(state.reason) or 'no reason recorded'}"
813
+ )
814
+ console.print(" [dim]Clear one with stage sources --clear <bucket>.[/dim]")
815
+
816
+ failing = report.failing_boards
817
+ if failing:
818
+ console.print()
819
+ console.print(f"[bold yellow]Boards needing a look[/bold yellow] ({len(failing)})")
820
+ shown_boards = failing if limit is None else failing[:limit]
821
+ for board in shown_boards:
822
+ console.print(
823
+ f" [yellow]{truncate(board.label, 32)}[/yellow] "
824
+ f"({board.source}) {board.consecutive_failures} consecutive failure(s) — "
825
+ f"{summary(board.last_error, 4000 if verbose else 48) or 'no error recorded'}"
826
+ )
827
+ if len(failing) > len(shown_boards):
828
+ console.print(
829
+ f" [dim]… and {len(failing) - len(shown_boards)} more; --all lists every row[/dim]"
830
+ )
831
+ console.print(
832
+ "[dim]These are registry rows to fix or switch off."
833
+ + ("" if verbose else " --verbose prints each error in full.")
834
+ + "[/dim]"
835
+ )
836
+
837
+ due = report.due_for_recheck
838
+ if due:
839
+ console.print()
840
+ console.print(f"[bold yellow]Registry rows due for re-check[/bold yellow] ({len(due)})")
841
+ shown_due = due if limit is None else due[:limit]
842
+ for entry in shown_due:
843
+ console.print(f" [yellow]{truncate(entry, 60)}[/yellow]")
844
+ if len(due) > len(shown_due):
845
+ console.print(
846
+ f" [dim]… and {len(due) - len(shown_due)} more; --all lists every row[/dim]"
847
+ )
848
+ console.print("[dim]Read each note and re-check before deciding.[/dim]")
849
+
850
+ console.print()
851
+ if not report.is_healthy:
852
+ console.print("[red]Problems above need attention.[/red]")
853
+ elif report.warnings:
854
+ console.print(f"[yellow]No errors, {report.warnings} warning(s).[/yellow]")
855
+ else:
856
+ console.print("[green]Healthy.[/green]")
857
+
858
+
859
+ def render_stats(
860
+ console: Console,
861
+ report: "StatsReport",
862
+ now: datetime,
863
+ *,
864
+ limit: int | None = BOARD_PREVIEW,
865
+ ) -> None:
866
+ console.print(
867
+ f"[bold]{report.total_jobs}[/bold] canonical posting(s), "
868
+ f"{report.duplicates} linked as duplicate(s), "
869
+ f"[bold]{sum(report.quarantined.values())}[/bold] quarantined, "
870
+ f"{report.tombstones} tombstone(s), {report.cached_urls} cached validator(s), "
871
+ f"schema v{report.schema_version}"
872
+ )
873
+ console.print()
874
+
875
+ if not report.runs:
876
+ console.print("[dim]No sync runs recorded yet — run stage sync.[/dim]")
877
+ else:
878
+ table = Table(box=None, pad_edge=False, header_style="bold", title="Recent syncs")
879
+ table.title_justify = "left"
880
+ table.add_column("when")
881
+ table.add_column("outcome")
882
+ table.add_column("elapsed", justify="right")
883
+ table.add_column("added", justify="right")
884
+ table.add_column("closed", justify="right")
885
+ table.add_column("quarantined", justify="right")
886
+ table.add_column("requests", justify="right")
887
+ table.add_column("cache", justify="right")
888
+ for run in report.runs:
889
+ requests = sum(stats.requests for stats in run.sources)
890
+ cached = sum(stats.not_modified for stats in run.sources)
891
+ elapsed = (run.finished_at - run.started_at).total_seconds()
892
+ colour = {"success": "green", "partial": "yellow"}.get(run.outcome.value, "red")
893
+ table.add_row(
894
+ _ago(run.finished_at, now),
895
+ f"[{colour}]{run.outcome.value}[/{colour}]",
896
+ f"{elapsed:.1f}s",
897
+ str(sum(stats.added for stats in run.sources)),
898
+ str(sum(stats.closed for stats in run.sources)),
899
+ str(sum(stats.quarantined for stats in run.sources)),
900
+ str(requests),
901
+ _ratio(cached / requests if requests else None),
902
+ )
903
+ console.print(table)
904
+
905
+ for column, counts in report.composition.items():
906
+ if not counts:
907
+ continue
908
+ console.print()
909
+ console.print(f"[bold]{column}[/bold]")
910
+ total = sum(counts.values())
911
+ listed = list(counts.items())
912
+ shown = listed if limit is None else listed[:limit]
913
+ for bucket, count in shown:
914
+ share = f"{count / total:.1%}" if total else "—"
915
+ console.print(f" {truncate(bucket, 24):26} {count:>6} [dim]{share}[/dim]")
916
+ if len(listed) > len(shown):
917
+ console.print(
918
+ f" [dim]… and {len(listed) - len(shown)} more; --all lists every row[/dim]"
919
+ )
920
+
921
+
922
+ def render_quarantine(
923
+ console: Console,
924
+ entries: Sequence[QuarantinedJob],
925
+ *,
926
+ total_matching: int,
927
+ reason_counts: dict[str, int],
928
+ ) -> None:
929
+ if not entries:
930
+ console.print(
931
+ "[yellow]Nothing quarantined.[/yellow] Rejections appear here after "
932
+ "[bold]stage sync[/bold]."
933
+ )
934
+ return
935
+
936
+ narrow = console.width < NARROW_COLUMNS
937
+ seen_width = 0 if narrow else 5
938
+ company_width = 14 if narrow else 16
939
+ location_width = 0 if narrow else 14
940
+ reason_width = 18 if narrow else 22
941
+ used = seen_width + company_width + location_width + reason_width
942
+ columns = 3 if narrow else 5
943
+ title_width = max(16, console.width - used - 2 * (columns + 1))
944
+
945
+ table = Table(box=None, pad_edge=False, header_style="bold")
946
+ if not narrow:
947
+ table.add_column("Seen", width=seen_width, no_wrap=True)
948
+ table.add_column("Company", width=company_width, no_wrap=True, overflow="ellipsis")
949
+ table.add_column("Title", width=title_width, no_wrap=True, overflow="ellipsis")
950
+ if not narrow:
951
+ table.add_column("Location", width=location_width, no_wrap=True, overflow="ellipsis")
952
+ table.add_column("Rejected for", width=reason_width, no_wrap=True, overflow="ellipsis")
953
+
954
+ for entry in entries:
955
+ title = clipped(entry.title_raw, title_width, style="dim")
956
+ _link(title, entry.apply_url_raw)
957
+ matched = f"{entry.reason.value}"
958
+ if entry.matched_phrase:
959
+ matched += f" ({truncate(entry.matched_phrase, 24)})"
960
+ cells = []
961
+ if not narrow:
962
+ cells.append(Text(entry.first_seen.astimezone().strftime("%m-%d")))
963
+ cells.append(Text(truncate(entry.company, 22)))
964
+ cells.append(title)
965
+ if not narrow:
966
+ cells.append(Text(truncate(place(entry.location_raw) or "—", 26)))
967
+ cells.append(Text(matched, style="yellow"))
968
+ table.add_row(*cells)
969
+
970
+ console.print(table)
971
+ shown = len(entries)
972
+ suffix = f" of {total_matching}" if total_matching > shown else ""
973
+ console.print(f"\n[dim]{shown} rejected posting(s){suffix}.[/dim]")
974
+ if reason_counts:
975
+ breakdown = ", ".join(f"{reason} {count}" for reason, count in reason_counts.items())
976
+ console.print(f"[dim]Across the whole table: {breakdown}.[/dim]")
977
+
978
+
979
+ def _empty_state(
980
+ window_days: int | None,
981
+ last_sync_at: datetime | None,
982
+ now: datetime,
983
+ hint: str = "",
984
+ ) -> str:
985
+ if last_sync_at is None:
986
+ return (
987
+ "[yellow]No postings yet.[/yellow] The database is empty — run [bold]stage sync[/bold] "
988
+ "to fetch from the registry."
989
+ )
990
+ age = (now - last_sync_at).days
991
+ when = "today" if age == 0 else f"{age} day(s) ago"
992
+ window = f" in the last {window_days} days" if window_days is not None else ""
993
+ if hint:
994
+ return f"[yellow]No matching postings{window}.[/yellow] {hint}"
995
+ return (
996
+ f"[yellow]No matching postings{window}.[/yellow] Widen the window with "
997
+ f"[bold]--last[/bold], relax a filter, or run [bold]stage sync[/bold] "
998
+ f"(last sync: {when})."
999
+ )
1000
+
1001
+
1002
+ async def render_sync(
1003
+ console: Console,
1004
+ events: AsyncIterator[SyncEvent],
1005
+ *,
1006
+ request_log: TextIO | None = None,
1007
+ progress: Callable[[SyncEvent], None] | None = None,
1008
+ ) -> SyncOutcome:
1009
+ outcome = SyncOutcome.FAILURE
1010
+ failures: list[tuple[str, str, str]] = []
1011
+ planned = 0
1012
+ validated = 0
1013
+ total = 0
1014
+ done = 0
1015
+
1016
+ def step() -> str:
1017
+ return f"[dim]{done:>4}/{total}[/dim] " if total else ""
1018
+
1019
+ async for event in events:
1020
+ if progress is not None:
1021
+ progress(event)
1022
+ match event:
1023
+ case SyncStarted(sources=sources, companies=companies):
1024
+ total = companies
1025
+ source_names = ", ".join(sources)
1026
+ console.print(
1027
+ f"[bold]Syncing[/bold] {companies} company board(s) via {source_names}"
1028
+ )
1029
+ case UnroutableCompanies(companies=stranded, platforms=platforms):
1030
+ console.print(
1031
+ f"\n[bold red]No adapter for {', '.join(platforms)}[/bold red] — "
1032
+ f"{len(stranded)} enabled row(s) will never be fetched: "
1033
+ f"{truncate(', '.join(stranded), 70)}"
1034
+ )
1035
+ console.print(
1036
+ " [dim]Set [bold]enabled: false[/bold] on those rows to record the gap "
1037
+ "without failing the run.[/dim]"
1038
+ )
1039
+ case BucketPlan() as bound:
1040
+ shared = f" ({', '.join(bound.sources)})" if len(bound.sources) > 1 else ""
1041
+ open_tag = "[yellow]" if bound.exceeds_ceiling else "[dim]"
1042
+ close_tag = "[/yellow]" if bound.exceeds_ceiling else "[/dim]"
1043
+ if bound.exceeds_ceiling and bound.planned > 1:
1044
+ detail = (
1045
+ f", worst case {bound.worst_case} against a ceiling of "
1046
+ f"{bound.ceiling} (the ceiling stops the run early)"
1047
+ )
1048
+ else:
1049
+ detail = ""
1050
+ console.print(
1051
+ f" {open_tag}bucket {bound.bucket}{close_tag}{shared} — "
1052
+ f"{bound.planned} planned{detail}"
1053
+ )
1054
+ case SourceBlocked() as blocked:
1055
+ console.print(
1056
+ f"\n[bold yellow]{blocked.source} blocked[/bold yellow] — bucket "
1057
+ f"[bold]{blocked.bucket}[/bold] is throttled for another "
1058
+ f"{_duration(blocked.remaining_s)} "
1059
+ f"(clears {blocked.blocked_until:%Y-%m-%d %H:%M UTC})"
1060
+ )
1061
+ reason = summary(blocked.reason, 70) if blocked.reason else "unknown"
1062
+ console.print(
1063
+ f" [dim]{blocked.consecutive_failures} consecutive failure(s): "
1064
+ f"{reason}. Not fetched this run — clear it with "
1065
+ f"[bold]stage sources --clear {blocked.bucket}[/bold].[/dim]"
1066
+ )
1067
+ case SourceStarted(source=source, companies=companies):
1068
+ console.print(f"\n[bold cyan]{source}[/bold cyan] — {companies} board(s)")
1069
+ case SourceRotated() as rotated:
1070
+ phase = "cycle complete" if rotated.wrapped else f"resumes after {rotated.cursor}"
1071
+ console.print(
1072
+ f" [dim]rotating[/dim] — {rotated.deferred} board(s) deferred to a later "
1073
+ f"run on bucket {rotated.bucket} ({phase})"
1074
+ )
1075
+ case SourceCapped() as capped:
1076
+ console.print(
1077
+ f" [yellow]capped[/yellow] — {capped.spent} spent by {capped.source} in 24h "
1078
+ f"across its buckets, so this run may use "
1079
+ f"{capped.allowance} of {capped.ceiling} per bucket"
1080
+ )
1081
+ case SourceFresh() as fresh:
1082
+ console.print(
1083
+ f" [dim]fresh[/dim] — {fresh.skipped} refreshed within "
1084
+ f"{fresh.refresh_interval_h:.0f}h, {fresh.remaining} left"
1085
+ )
1086
+ case SourceResting() as resting:
1087
+ console.print(
1088
+ f" [dim]resting[/dim] — {resting.skipped} board(s) backing off after "
1089
+ f"repeated failures, {resting.remaining} left"
1090
+ )
1091
+ case PlannedRequest(company=company, url=url, has_validator=cached):
1092
+ if not planned:
1093
+ console.print(
1094
+ " [dim]cached = validator on file, a 304 is expected; "
1095
+ "cold = full response expected[/dim]"
1096
+ )
1097
+ planned += 1
1098
+ validated += int(cached)
1099
+ cache_marker = "[cyan]cached[/cyan]" if cached else "[yellow]cold [/yellow]"
1100
+ room = max(20, console.width - 34)
1101
+ console.print(f" {cache_marker} {truncate(company, 22):<22} {truncate(url, room)}")
1102
+ case RequestLogged() as record:
1103
+ _write_request_log(request_log, record)
1104
+ case CompanyFinished(
1105
+ company=company, fetched=fetched, elapsed_ms=elapsed, degraded=degraded
1106
+ ):
1107
+ status_marker = "[yellow]part[/yellow]" if degraded else "[green]ok[/green] "
1108
+ done += 1
1109
+ console.print(
1110
+ f" {step()}{status_marker} {sanitize(company):<28} "
1111
+ f"{fetched:>4} posting(s) {elapsed:>7.0f}ms"
1112
+ )
1113
+ if degraded:
1114
+ console.print(f" [yellow]{truncate(degraded, 88)}[/yellow]")
1115
+ case CompanyUnchanged(company=company, elapsed_ms=elapsed):
1116
+ done += 1
1117
+ console.print(
1118
+ f" {step()}[cyan]304[/cyan] {sanitize(company):<28} "
1119
+ f"{'unchanged':>14} {elapsed:>7.0f}ms"
1120
+ )
1121
+ case CompanyFailed(source=source, company=company, error=error, elapsed_ms=elapsed):
1122
+ failures.append((source, company, error))
1123
+ done += 1
1124
+ console.print(
1125
+ f" {step()}[red]fail[/red] {sanitize(company):<28} "
1126
+ f"{summary(error, 60)} {elapsed:>7.0f}ms"
1127
+ )
1128
+ case CompanyDeferred(company=company):
1129
+ done += 1
1130
+ console.print(
1131
+ f" {step()}[yellow]budget[/yellow] {sanitize(company):<26} "
1132
+ f"{'not attempted, deferred to the next run':>40}"
1133
+ )
1134
+ case SourceFailed(source=source, error=error):
1135
+ failures.append((source, source, error))
1136
+ console.print(f" [red]fail[/red] {sanitize(source)}: {summary(error, 60)}")
1137
+ case SourceFinished() as finished:
1138
+ _render_source_summary(console, finished)
1139
+ case SyncFinished(dry_run=True) as finished:
1140
+ outcome = finished.outcome
1141
+ console.print(
1142
+ f"\n[bold]dry run[/bold] — {planned} request(s) planned, none sent. "
1143
+ f"{validated} carry a validator, so a real run would likely transfer "
1144
+ f"{planned - validated} full response(s)."
1145
+ )
1146
+ if outcome is not SyncOutcome.SUCCESS:
1147
+ console.print(
1148
+ "[red]Pre-flight failed[/red] on a fault that needs no network to "
1149
+ "see. Fix it before running the real sync."
1150
+ )
1151
+ case SyncFinished() as finished:
1152
+ outcome = finished.outcome
1153
+ cache_note = ""
1154
+ if finished.requests:
1155
+ cache_note = f", {finished.not_modified}/{finished.requests} cached"
1156
+ if finished.requests >= CACHE_RATIO_MINIMUM:
1157
+ ratio = finished.not_modified / finished.requests
1158
+ cache_note += f" ({ratio:.0%})"
1159
+ purge_note = f", {finished.purged} purged" if finished.purged else ""
1160
+ quarantine_note = ""
1161
+ if finished.quarantined:
1162
+ quarantine_note = f", [yellow]{finished.quarantined} quarantined[/yellow]"
1163
+ reason_note = (
1164
+ f" [dim]({sanitize(finished.partial_reason)})[/dim]"
1165
+ if finished.partial_reason
1166
+ else ""
1167
+ )
1168
+ console.print(
1169
+ f"\n[bold]{finished.outcome.value}[/bold]{reason_note} — "
1170
+ f"{finished.added} added, {finished.updated} updated, "
1171
+ f"{finished.closed} closed"
1172
+ f"{quarantine_note}{purge_note}{cache_note}"
1173
+ )
1174
+ if finished.quarantined:
1175
+ console.print(
1176
+ "[dim]Rejected postings are kept. Review them with "
1177
+ "[bold]stage quarantine[/bold].[/dim]"
1178
+ )
1179
+ case _:
1180
+ continue
1181
+
1182
+ if failures:
1183
+ console.print("\n[bold red]Failed sources[/bold red]")
1184
+ for source, company, error in failures:
1185
+ console.print(f" {source}/{sanitize(company)}: {first_line(error)}")
1186
+ console.print(
1187
+ "\n[dim]Re-run one source with [bold]stage sync --source <name>[/bold] "
1188
+ "to reproduce.[/dim]"
1189
+ )
1190
+ return outcome
1191
+
1192
+
1193
+ _VERDICT_STYLE = {
1194
+ ProbeVerdict.MATCH: ("green", "match"),
1195
+ ProbeVerdict.UNVERIFIED: ("yellow", "check"),
1196
+ ProbeVerdict.REJECTED: ("red", "reject"),
1197
+ ProbeVerdict.EMPTY: ("dim", "empty"),
1198
+ ProbeVerdict.ERROR: ("red", "error"),
1199
+ ProbeVerdict.MISS: ("dim", "miss"),
1200
+ }
1201
+
1202
+
1203
+ async def render_discovery(
1204
+ console: Console,
1205
+ events: AsyncIterator[DiscoveryEvent],
1206
+ *,
1207
+ verified_on: date | None = None,
1208
+ display_name: str | None = None,
1209
+ request_log: TextIO | None = None,
1210
+ collect: bool = False,
1211
+ progress: Callable[[DiscoveryEvent], None] | None = None,
1212
+ ) -> bool | DiscoveryFinished:
1213
+ from stage.companies import registry_entry_yaml
1214
+ from stage.services.discover import to_company
1215
+
1216
+ resolved = False
1217
+ outcome: DiscoveryFinished | None = None
1218
+
1219
+ def show_entry(name: str, candidate: PlatformCandidate, note: str) -> None:
1220
+ console.print(f"\n[bold green]{sanitize(name)}[/bold green] -> {candidate.label}")
1221
+ if note:
1222
+ console.print(f" [dim]{sanitize(note)}[/dim]")
1223
+ letter = next((c for c in name.lower() if c.isalpha()), "a")
1224
+ console.print(f"\n[dim]Paste into src/stage/data/companies/{letter}.yaml:[/dim]")
1225
+ entry = registry_entry_yaml(to_company(name, candidate, verified_on=verified_on))
1226
+ for line in entry.splitlines():
1227
+ console.print(plain(f" {line}"), highlight=False)
1228
+
1229
+ async for event in events:
1230
+ if progress is not None:
1231
+ progress(event)
1232
+ match event:
1233
+ case DiscoveryStarted(companies=names, platforms=platforms, probes_planned=planned):
1234
+ console.print(
1235
+ f"[bold]Probing[/bold] {len(names)} name(s) across {len(platforms)} "
1236
+ f"platform(s) — {planned} probe(s), plus one board-metadata request "
1237
+ "per positive"
1238
+ )
1239
+ console.print(
1240
+ " [dim]Slug guessing resolves ~16%, a third of them falsely. "
1241
+ "Prefer [bold]--url[/bold].[/dim]"
1242
+ )
1243
+ case RequestLogged() as record:
1244
+ _write_request_log(request_log, record)
1245
+ case CandidateSkipped(company=company, slug=slug, reason=reason):
1246
+ console.print(
1247
+ f" [dim]skip[/dim] {truncate(company, 22):<22} "
1248
+ f"[dim]{sanitize(slug)} — {sanitize(reason)}[/dim]"
1249
+ )
1250
+ case UrlResolved(candidate=candidate, detail=detail):
1251
+ resolved = True
1252
+ note = detail
1253
+ if display_name is None:
1254
+ note = (
1255
+ f"{detail + '. ' if detail else ''}Set the display name with "
1256
+ "--name — the slug is a board token, not a company name"
1257
+ )
1258
+ show_entry(display_name or candidate.slug, candidate, note)
1259
+ case UrlUnrecognized(url=url, detail=detail):
1260
+ console.print(f"\n[yellow]Unrecognized[/yellow] {truncate(url, 70)}")
1261
+ console.print(f" {sanitize(detail)}")
1262
+ _show_custom_skeleton(console, url, display_name)
1263
+ case PlatformProbed(result=result) if result.verdict is not ProbeVerdict.MISS:
1264
+ style, label = _VERDICT_STYLE[result.verdict]
1265
+ count = "" if result.job_count is None else f"{result.job_count:>5} job(s)"
1266
+ console.print(
1267
+ f" [{style}]{label:<6}[/{style}] {truncate(result.company, 22):<22} "
1268
+ f"{result.candidate.label:<34} {count}"
1269
+ )
1270
+ if result.detail:
1271
+ console.print(f" [dim]{truncate(result.detail, 88)}[/dim]")
1272
+ case DiscoveryFinished() as finished:
1273
+ outcome = finished
1274
+ resolved = resolved or bool(finished.matched)
1275
+ _render_discovery_summary(console, finished, show_entry, quiet=collect)
1276
+ case _:
1277
+ continue
1278
+ return outcome if collect and outcome is not None else resolved
1279
+
1280
+
1281
+ def _show_custom_skeleton(console: Console, url: str, display_name: str | None) -> None:
1282
+ target = web_url(url)
1283
+ if target is None:
1284
+ return
1285
+ console.print(
1286
+ "\n[dim]If that page fills itself in from a JSON request, this is "
1287
+ "[bold]custom_json[/bold]. Open DevTools, filter Fetch/XHR, reload the page, and copy "
1288
+ "the request that returns the job list. Then paste this into the registry file for that "
1289
+ "company's first letter, under src/stage/data/companies/, with the field names taken "
1290
+ "from that response:[/dim]"
1291
+ )
1292
+ for line in (
1293
+ f"- name: {display_name or 'REPLACE ME'}",
1294
+ " platform: custom_json",
1295
+ f" slug: {registry_slug(display_name or target)}",
1296
+ " enabled: false",
1297
+ " custom:",
1298
+ " url: PASTE_THE_JSON_REQUEST_URL_HERE",
1299
+ " jobs_path: data.jobs",
1300
+ " fields:",
1301
+ " id: id",
1302
+ " title: title",
1303
+ " location: location",
1304
+ " url: absoluteUrl",
1305
+ ):
1306
+ console.print(plain(f" {line}"), highlight=False)
1307
+
1308
+
1309
+ def registry_slug(value: str) -> str:
1310
+ from stage.lexicon import fold
1311
+
1312
+ return "-".join(fold(value).split())[:40] or "replace-me"
1313
+
1314
+
1315
+ def _render_discovery_summary(
1316
+ console: Console,
1317
+ event: DiscoveryFinished,
1318
+ show_entry: Callable[[str, PlatformCandidate, str], None],
1319
+ quiet: bool = False,
1320
+ ) -> None:
1321
+ for warning in event.ceiling_hit:
1322
+ console.print(
1323
+ f"\n[bold red]Per-host ceiling reached[/bold red] — {sanitize(warning)}. "
1324
+ "Probing stopped for that platform; split the batch across runs."
1325
+ )
1326
+ for platform, count in event.non_json:
1327
+ console.print(
1328
+ f"\n[yellow]{platform}: all {count} probe(s) returned non-JSON.[/yellow] "
1329
+ "Open one URL by hand before trusting a miss here."
1330
+ )
1331
+ console.print(
1332
+ f"\n[bold]{len(event.matched)} match(es)[/bold], {len(event.unverified)} needing a "
1333
+ f"manual check, {len(event.rejected)} rejected, {event.missed} miss(es), "
1334
+ f"{event.errors} error(s) in {event.requests} request(s) "
1335
+ f"({event.elapsed_ms / 1000:.1f}s)"
1336
+ )
1337
+ for result in () if quiet else event.matched:
1338
+ show_entry(result.company, result.candidate, f"board name {result.board_name!r} confirmed")
1339
+ if event.unverified:
1340
+ console.print(
1341
+ "\n[yellow]Verify before adding.[/yellow] These expose no board name, "
1342
+ "and a 200 with jobs is not evidence of the right company."
1343
+ )
1344
+ for result in event.unverified:
1345
+ console.print(plain(f" {result.company} -> {result.candidate.label} {result.url}"))
1346
+ if not event.matched and not event.unverified:
1347
+ console.print(
1348
+ "\n[dim]Nothing resolved. Re-run with "
1349
+ "[bold]stage discover --url <careers-page>[/bold], the only path that "
1350
+ "resolves Workday.[/dim]"
1351
+ )
1352
+
1353
+
1354
+ def _render_source_summary(console: Console, event: SourceFinished) -> None:
1355
+ parts = [
1356
+ f"{event.added} added",
1357
+ f"{event.updated} updated",
1358
+ f"{event.closed} closed",
1359
+ ]
1360
+ if event.quarantined:
1361
+ parts.append(f"[yellow]{event.quarantined} quarantined[/yellow]")
1362
+ if event.requests:
1363
+ parts.append(f"{event.not_modified}/{event.requests} cached")
1364
+ parts.append(f"p50 {event.latency_p50_ms:.0f}ms")
1365
+ parts.append(f"p95 {event.latency_p95_ms:.0f}ms")
1366
+ if event.fetch_ms or event.normalize_ms or event.write_ms:
1367
+ parts.extend(
1368
+ (
1369
+ f"fetch {event.fetch_ms / 1000:.1f}s",
1370
+ f"process {event.normalize_ms / 1000:.1f}s",
1371
+ f"write {event.write_ms / 1000:.1f}s",
1372
+ )
1373
+ )
1374
+ if event.retries:
1375
+ parts.append(f"{event.retries} retried")
1376
+ if event.tightenings:
1377
+ parts.append(f"[yellow]{event.tightenings} rate tightening(s)[/yellow]")
1378
+ console.print(f" [dim]{', '.join(parts)} in {event.elapsed_ms / 1000:.1f}s[/dim]")
1379
+
1380
+
1381
+ def _write_request_log(stream: TextIO | None, record: RequestLogged) -> None:
1382
+ if stream is None:
1383
+ return
1384
+ stream.write(
1385
+ json.dumps(
1386
+ {
1387
+ "source": record.source,
1388
+ "method": record.method,
1389
+ "url": record.url,
1390
+ "status": record.status,
1391
+ "elapsed_ms": round(record.elapsed_ms, 2),
1392
+ "attempt": record.attempt,
1393
+ "error": record.error,
1394
+ },
1395
+ ensure_ascii=False,
1396
+ )
1397
+ + "\n"
1398
+ )