atsfill 0.0.7.dev211__tar.gz → 0.0.8.dev219__tar.gz

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atsfill
-Version: 0.0.7.dev211
+Version: 0.0.8.dev219
 Summary: Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle.
 Keywords: job-applications,ats,ashby,greenhouse,teamtailor,ai-agents,cli
 Author: shbernal
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Office/Business
 Classifier: Typing :: Typed
-Requires-Dist: camoufox==0.5.8
+Requires-Dist: camoufox==0.5.9b6
 Requires-Dist: filelock==4.1.1
 Requires-Dist: keyring==25.7.0
 Requires-Dist: packaging==26.3

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.7.dev211"
+version = "0.0.8.dev219"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"
@@ -28,7 +28,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "camoufox==0.5.8",
+    "camoufox==0.5.9b6",
     "filelock==4.1.1",
     "keyring==25.7.0",
     "packaging==26.3",

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.7.dev211"
+version = "0.0.8.dev219"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"
@@ -21,7 +21,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "camoufox==0.5.8",
+    "camoufox==0.5.9b6",
     "filelock==4.1.1",
     "keyring==25.7.0",
     "packaging==26.3",

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/breezy.py

@@ -328,6 +328,11 @@ class Breezy:
             case _:
                 controls.first.fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The controls `browser_read` names; a work history or education entry's have no name.
+        Every step stays in the page, the others hidden."""
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         form = page.locator(FORM)
         out: dict[str, Answer] = {}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/greenhouse.py

@@ -11,13 +11,14 @@ import re
 import time
 import urllib.request
 from contextlib import suppress
+from dataclasses import replace
 from typing import Any
 from urllib.parse import parse_qs, urlencode, urlsplit
 
 from atsfill.core import browser, otp
 from atsfill.core.adapter import Blocked, Gap, Unsupported
 from atsfill.core.browser import HTTP_TIMEOUT_S, BrowserError, Page, Timeout
-from atsfill.core.results import Answer, Field, FieldType, SideEffect
+from atsfill.core.results import Answer, Field, FieldType, Is, SideEffect
 
 HOSTS = re.compile(r"^(job-boards|boards)(\.eu)?\.greenhouse\.io$")
 FORM = "#application-form"
@@ -29,6 +30,7 @@ EDUCATION_API = "https://boards-api.greenhouse.io/v1/boards/{board}/education/{k
 EMBED = "https://job-boards.greenhouse.io/embed/job_app"
 EMBED_TIMEOUT_MS = 15_000
 OPTIONS_TIMEOUT_MS = 3_000
+RACE_TIMEOUT_MS = 3_000
 HYDRATE_TIMEOUT_MS = 15_000
 FORM_TIMEOUT_MS = 20_000
 ANSWER_TIMEOUT_MS = 15_000  # an upload's file name, a suggestion: past it, a mismatch
@@ -52,6 +54,7 @@ OTP_ERROR = "#email-verification-error"
 # Sections the page adds or names its own way, or shows a question of only on another answer
 # (EEOC race after Hispanic/Latino): `request_read` declines a posting with any (ADR 0011).
 UNMAPPED = ("education", "compliance", "demographic_questions")
+ETHNICITY, NOT_HISPANIC, RACE = "hispanic_ethnicity", "No", "race"
 
 # The fields in page order. A select's options are read by opening it, from Python.
 _READ = """form => {
@@ -196,10 +199,11 @@ class Greenhouse:
         fields = [self._field(page, raw) for raw in raws]
         self._fields = {f.id: f for f in fields}
         self._kinds = {raw["id"]: raw["kind"] for raw in raws}
-        return fields
+        return self._with_race(page, fields)
 
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
-        for field_id, answer in answers.items():
+        # the page draws race only once the answer to Hispanic/Latino is in
+        for field_id, answer in sorted(answers.items(), key=lambda a: a[0] == RACE):
             values = [answer] if isinstance(answer, str) else answer
             match self._kinds[field_id]:
                 case "file":
@@ -235,6 +239,9 @@ class Greenhouse:
                 case _:
                     page.locator(_by_id(field_id)).fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, list(self._fields))
         return {k: v for k, v in values.items() if v not in (None, "", [])}
@@ -295,6 +302,30 @@ class Greenhouse:
             one_of=raw.get("oneOf"),
         )
 
+    def _with_race(self, page: Page, fields: list[Field]) -> list[Field]:
+        """`fields` with the EEOC race question, a follow-up of Hispanic/Latino: its page's own
+        label and options show only with the question, so the read answers No, reads it and
+        clears the answer. The API's options are not the page's."""
+        ethnicity = self._fields.get(ETHNICITY)
+        if (
+            RACE in self._fields
+            or ethnicity is None
+            or NOT_HISPANIC not in (ethnicity.options or [])
+        ):
+            return fields
+        _select(page, ETHNICITY, NOT_HISPANIC, False)
+        with suppress(Timeout):
+            page.locator(_by_id(RACE)).wait_for(timeout=RACE_TIMEOUT_MS)
+        raws = page.locator(FORM).evaluate(_READ)
+        raw = next((r for r in raws if r["id"] == RACE), None)
+        if raw is not None:
+            race = replace(self._field(page, raw), shown_when=Is(ETHNICITY, [NOT_HISPANIC]))
+            self._fields[RACE], self._kinds[RACE] = race, raw["kind"]
+            fields = [*fields]
+            fields.insert(fields.index(ethnicity) + 1, race)
+        _clear(page, ETHNICITY)
+        return fields
+
     def _education(self, page: Page, field_id: str) -> list[str] | None:
         """An education select's options, every page of them, read once per kind; None for
         another field or when the endpoint does not answer."""
@@ -429,6 +460,14 @@ def _exact(value: str) -> re.Pattern[str]:
     return re.compile(rf"^\s*{re.escape(value)}\s*$")
 
 
+def _clear(page: Page, field_id: str) -> None:
+    """Empties a single react-select."""
+    box = page.locator(_by_id(field_id))
+    box.click()
+    box.press("Backspace")
+    box.press("Escape")
+
+
 def _select(page: Page, field_id: str, value: str, suggested: bool) -> None:
     """Types into a react-select and picks the option: focused with the arrow keys then Enter,
     which every select takes (a hover does not move the focus once typed). A suggestion list

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/hiringthing.py

@@ -16,7 +16,7 @@ from urllib.parse import urlsplit
 
 from atsfill.core.adapter import Blocked, Gap, NotTaken, Unsupported
 from atsfill.core.browser import Locator, Page, Timeout
-from atsfill.core.results import Answer, Field, FieldType, SideEffect
+from atsfill.core.results import Answer, Field, FieldType, Is, SideEffect
 
 # The white-label domains HiringThing serves its career sites on, as freehire indexes them.
 DOMAINS = (
@@ -56,6 +56,9 @@ SUBMIT_TIMEOUT_MS = 90_000
 REQUIRED = re.compile(r"\s*\(required\)\s*$")
 DATE = "%m/%d/%Y"
 YES_NO = ["Yes", "No"]
+SOURCE = "app.where_from"
+SOURCE_DETAIL = "app.where_from_other"  # rendered only once a source below is picked
+DETAILED_SOURCES = ("Friend/Colleague", "Other")
 TYPES = {
     "text": FieldType.TEXT,
     "date": FieldType.DATE,
@@ -176,10 +179,19 @@ class HiringThing:
         questions = _questions(props.get_attribute("data-react-props") or "{}")
         controls = form.evaluate(_CONTROLS)
         self._kinds = {c["id"]: c["kind"] for c in controls if c["kind"] != "unhandled"}
-        return [
+        fields = [
             _field(c, questions, _menu(page, c["id"]) if c["kind"] == "select" else None)
             for c in controls
         ]
+        source = next((f for f in fields if f.id == SOURCE), None)
+        if source and (detailed := [o for o in source.options or [] if o in DETAILED_SOURCES]):
+            self._kinds[SOURCE_DETAIL] = "text"
+            at = fields.index(source) + 1
+            fields.insert(at, _source_detail(detailed))
+        return fields
+
+    def shown_fields(self, page: Page) -> list[str]:
+        return [c["id"] for c in page.locator(FORM).evaluate(_CONTROLS)]
 
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
         form = page.locator(FORM)
@@ -275,6 +287,12 @@ def _field(control: dict, questions: dict[str, str], options: list[str] | None)
     return Field(field_id, label, TYPES[kind], required, options, max_length)
 
 
+def _source_detail(sources: list[str]) -> Field:
+    """What the page asks once one of `sources` is picked as the referral source."""
+    label = "Please Specify Source"
+    return Field(SOURCE_DETAIL, label, FieldType.TEXT, True, shown_when=Is(SOURCE, sources))
+
+
 def _combobox(page: Page, field_id: str) -> Locator:
     return page.locator(f"{FORM} input[role=combobox][id='{field_id}']")
 

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/jazzhr.py

@@ -190,6 +190,9 @@ class JazzHR:
                 case _:
                     field.fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, self._fields)
         return {k: v for k, v in values.items() if v not in (None, "", [])}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/jobaffinity.py

@@ -68,6 +68,19 @@ _READ_BACK = """(form, fields) => {
   }
   return out;
 }"""
+# The fields a form shows, by the names `_parse` gives them: its controls, a radio group once,
+# and its documents, the older form's in forms of their own beside it.
+_SHOWN = """form => {
+  const shown = el => el.getClientRects().length > 0;
+  const skipped = ["hidden", "submit", "button", "file"];
+  const names = [...form.querySelectorAll("input, select, textarea")]
+    .filter(el => el.name && !skipped.includes(el.type) && shown(el)).map(el => el.name);
+  const newer = [...form.querySelectorAll("[data-form-field-name]")]
+    .filter(shown).map(el => el.dataset.formFieldName);
+  const older = [...document.querySelectorAll("form[id^='js-'][id$='-form']")]
+    .filter(shown).map(el => el.id.slice("js-".length, -"-form".length));
+  return [...new Set([...names, ...newer, ...older])];
+}"""
 
 
 @dataclass
@@ -152,6 +165,12 @@ class JobAffinity:
                 if c.questionnaire:
                     _set(questions, c, answers[c.field.id])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        shown = page.locator(NEW if self._form.new else OLD).evaluate(_SHOWN)
+        if self._questions:
+            shown += self._questions.locator(QUESTIONNAIRE).evaluate(_SHOWN)
+        return shown
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         main = [c for c in self._form.controls if not c.questionnaire]
         values = _values(page, NEW if self._form.new else OLD, main)

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/jobstreet.py

@@ -92,6 +92,10 @@ _QUESTIONS = """() => {
 }"""
 
 
+# The questions the employer's step shows, by control name.
+_ASKED = """() => [...document.querySelectorAll('[name^="questionnaire."]')].map(e => e.name)"""
+
+
 class Jobstreet:
     name = "jobstreet"
     per_employer = True  # one SEEK account, signed in on each site apart: the scope is the site
@@ -105,6 +109,7 @@ class Jobstreet:
     ]
 
     def __init__(self) -> None:
+        self._asked: list[str] = []  # the employer's step's control names, once answered
         self._fields: dict[str, Field] = {}
         self._read: dict[str, Answer] = {}
 
@@ -198,6 +203,7 @@ class Jobstreet:
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
         """Walks the steps in order, each answered and left with Continue, which stays put with
         its errors while one is missing; the review step takes the profile's visibility."""
+        self._asked = []
         self._read = {k: f.prefilled for k in DETAILS if (f := self._fields[k]).prefilled}
         self._read |= _personal_details(page, self._fields, answers)
         _documents_fill(page, answers)
@@ -208,6 +214,7 @@ class Jobstreet:
             for field_id, answer in answers.items():
                 if field_id.startswith("question."):
                     _answer(page, self._fields[field_id], answer)
+            self._asked = page.evaluate(_ASKED)
             asked = page.evaluate(_QUESTIONS)
             self._read |= {
                 f"question.{name.removeprefix('questionnaire.')}": value
@@ -232,6 +239,13 @@ class Jobstreet:
             if radio.is_checked():
                 self._read["profileVisibility"] = radio.get_attribute("value") or ""
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The questions the employer's step showed once answered: the wizard has left it. The
+        other steps' controls are the account's and the page's, read as `browser_read` lists
+        them."""
+        names = dict.fromkeys(self._asked)
+        return [f"question.{name.removeprefix('questionnaire.')}" for name in names]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         return dict(self._read)
 

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/join.py

@@ -151,6 +151,7 @@ class Join:
         self._fields: dict[str, Field] = {}
         self._questions: list[dict[str, Any]] = []
         self._seen: dict[str, Answer] = {}  # each question as its step was left
+        self._walked: dict[str, None] = {}  # the file and question fields of the steps walked
         self._files: list[str] = []  # the file fields answered, in the steps' order
 
     def detect(self, url: str, page: Page | None) -> bool:
@@ -253,15 +254,18 @@ class Join:
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
         """Walks the wizard from the step it is on to the review page, answering each step; a
         step without a listed field (the account's name, its links) is left as it is."""
+        self._walked = {}
         for _ in range(MAX_STEPS):
             step = _step(page)
             if step == "review":
                 page.locator("a > div > p + p").first.wait_for(timeout=STEP_TIMEOUT_MS)
                 return
             if step in FILES:
+                self._walked[FILES[step]] = None
                 self._file(page, FILES[step], answers.get(FILES[step]))
             elif question := self._step_question(step):
                 field_id = f"question.{question['id']}"
+                self._walked[field_id] = None
                 if field_id in self._fields:
                     self._question(page, question, answers.get(field_id))
             elif step in PROFILE_STEPS:
@@ -289,6 +293,10 @@ class Join:
             raise Unsupported(f"unhandled_rendering: a question step {step} not listed")
         return None
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The fields of the steps the fill walked: the wizard keeps only the step it is on."""
+        return list(self._walked)
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         """The files as the review page lists them; each question as its step was left, since
         the review shows a classed question's answer reworded (`€50,000`)."""

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/lever.py

@@ -7,6 +7,7 @@ id. Nothing but the resume reaches Lever before submit, so the read-back is the
 
 import re
 from contextlib import suppress
+from dataclasses import replace
 from datetime import date
 from typing import Any
 from urllib.parse import urlsplit
@@ -14,7 +15,7 @@ from urllib.parse import urlsplit
 from atsfill.core import browser
 from atsfill.core.adapter import Blocked, Gap, Unsupported
 from atsfill.core.browser import Locator, Page, Timeout
-from atsfill.core.results import Answer, Field, FieldType, SideEffect
+from atsfill.core.results import Answer, Field, FieldType, Is, SideEffect
 
 HOSTS = re.compile(r"^jobs\.(eu\.)?lever\.co$")
 JOB = re.compile(r"^/([^/]+)/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b")
@@ -26,6 +27,8 @@ SUGGEST_TIMEOUT_MS = 10_000
 SUBMIT_TIMEOUT_MS = 90_000
 KEY_DELAY_MS = 40
 YES_NO = ["Yes", "No"]
+DISABILITY = "eeo[disability]"
+SIGNATURE = ("eeo[disabilitySignature]", "eeo[disabilitySignatureDate]")  # shown by its answer
 KINDS = {
     "text": FieldType.TEXT,
     "phone": FieldType.PHONE,
@@ -38,7 +41,8 @@ KINDS = {
     "checkboxes": FieldType.MULTI_CHOICE,
 }
 
-# The fields in page order, one per control name: its question's label, and how it renders.
+# The fields in page order, one per control name: its question's label, how it renders, and
+# whether its question is on show.
 _READ = """form => {
   const text = el => {
     if (!el) return "";
@@ -62,7 +66,7 @@ _READ = """form => {
       const label = box.querySelector(".application-label .text") ||
         box.querySelector(".application-label");
       field = {id: name, label: text(label), required: !!box.querySelector(".required"),
-               options: [], count: 0};
+               options: [], count: 0, shown: box.getClientRects().length > 0};
       if (el.type === "file") {
         field.kind = "file";
       } else if (el.tagName === "SELECT") {
@@ -153,14 +157,20 @@ class Lever:
         except Timeout:
             raise Unsupported("posting_gone: no application form on the page") from None
         fields, self._kinds = [], {}
-        for raw in form.evaluate(_READ):
-            fields.append(_field(raw))
+        raws = form.evaluate(_READ)
+        asked = {raw["id"] for raw in raws}
+        for raw in raws:
+            field = _field(raw)
+            if raw["id"] in SIGNATURE and DISABILITY in asked:
+                field = replace(field, shown_when=Is(DISABILITY))
+            fields.append(field)
             self._kinds[raw["id"]] = raw["kind"]
         return fields
 
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
         # the resume is parsed into the form, so it goes first and is waited for
-        for name, answer in sorted(answers.items(), key=lambda a: a[0] != "resume"):
+        first = sorted(answers.items(), key=lambda a: a[0] != "resume")
+        for name, answer in sorted(first, key=lambda a: a[0] in SIGNATURE):
             values = [answer] if isinstance(answer, str) else answer
             controls = page.locator(f"{FORM} [name={_quoted(name)}]:not([type=hidden])")
             match self._kinds[name]:
@@ -183,6 +193,9 @@ class Lever:
                 case _:
                     controls.first.fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ) if raw["shown"]]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, self._kinds)
         for name, kind in self._kinds.items():

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/manatal.py

@@ -61,6 +61,23 @@ _READ = """fields => Object.fromEntries(fields.map(([id, kind, dom]) => {
     default: return [id, el ? el.value || null : null];
   }
 }))"""
+# The fields the form shows: a control by the field id of its element, else by its name, an
+# Other text box by its list. A radio or a box sits unrendered under its styled mark, and
+# select2 adds a search input of its own.
+_SHOWN = """fields => {
+  const form = document.querySelector("#submit-id-submit").closest("form");
+  const rendered = el => el.getClientRects().length > 0;
+  const out = new Set();
+  for (const el of form.querySelectorAll("input, select, textarea")) {
+    if (["hidden", "submit", "button"].includes(el.type)) continue;
+    if (el.matches(".select2-search__field")) continue;
+    if (!rendered(el) && !rendered(el.parentElement)) continue;
+    if (el.matches(".custom-other-val-input"))
+      out.add(`${el.closest("li").querySelector("input[name]").name}.other`);
+    else out.add(fields[el.id] ?? (el.name || el.id));
+  }
+  return [...out];
+}"""
 # Selects the options with these labels on a select2 multiple select, which select2 follows.
 _SELECT = """(el, labels) => {
   for (const o of el.options) o.selected = labels.includes(o.text.trim());
@@ -290,6 +307,9 @@ class Manatal:
                         f"unhandled_rendering: cannot fill {field_id}, a {kind}, with {answer!r}"
                     )
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return page.evaluate(_SHOWN, {dom: fid for fid, _, dom in _inputs(self._entries)})
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         inputs = [list(i) for i in _inputs(self._entries)]
         if page.locator(f'input[name="{TERMS}"]').count():

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/oracle.py

@@ -4,7 +4,8 @@ as docs/adapters/oracle.md observes.
 The form opens only past an email step, and an email the employer's Oracle already knows gets an
 emailed OTP first, typed in by the core. Reading passes the step with a throwaway address;
 filling starts over with the answer's. Oracle keeps a draft of the application from then on,
-autosaved as fields change.
+autosaved as fields change. The address follows the country's format, and which rows a country
+shows is known only once it is picked: the fill reveals those of a country other than the form's.
 """
 
 import re
@@ -42,6 +43,8 @@ ATTACHED = ".attachment-upload-button__download, .misc-attachment-list__item-nam
 REMOVE = ".attachment-upload-button__filled .attachment-upload-button__bottom-button"
 STEP_TIMEOUT_MS = 30_000
 LIST_TIMEOUT_MS = 3_000
+LIST_POLL_MS = 300  # between two looks at a list, which a search narrows as it answers
+LIST_POLLS = 10
 SEARCH_TIMEOUT_MS = 6_000  # for a typed search's results
 PAGE_TIMEOUT_MS = 15_000  # per click on a page number
 SUBMIT_TIMEOUT_MS = 90_000
@@ -115,7 +118,7 @@ _READ = """() => {
       const named = row.querySelector("[id^=labelText-]");
       const input = row.querySelector("input:not([type=hidden]), textarea");
       let id = stable(named?.id || label?.getAttribute("for") || label?.id || input?.name);
-      const field = {id, label: text(row.querySelector(
+      const field = {id, address: !!row.closest(".address-form"), label: text(row.querySelector(
           ".input-row__linebreak, .input-row__label-text") || label),
         required: !!label?.classList.contains("input-row__label--required")
           || !!row.querySelector("[aria-required=true], [required]")};
@@ -260,6 +263,8 @@ class Oracle:
         self._kinds: dict[str, str] = {}
         self._pages: dict[str, int] = {}  # field id: the page of the form it is on
         self._picked: dict[str, str] = {}  # a geography field's id: the item picked in it
+        self._left_out: set[str] = set()  # the geography levels the most specific one fills
+        self._filling = False  # past the email step, with the answers given so far
         self._months: list[str] = []
         self._job = ""
 
@@ -284,7 +289,7 @@ class Oracle:
         self._pass_email_step(
             page, f"atsfill.read.{uuid.uuid4().hex[:12]}@example.com", bool(legal)
         )
-        self._kinds, self._pages = {}, {}
+        self._kinds, self._pages, self._left_out, self._filling = {}, {}, set(), False
         # The page numbers move between pages without checking the page left.
         for number in range(1, max(len(_page_numbers(page)), 1) + 1):
             _go_to(page, number)
@@ -293,13 +298,40 @@ class Oracle:
                 for raw in page.evaluate(_READ)
                 if raw["id"] not in self._kinds and (field := self._described(page, raw))
             ]
-            for raw, field in _geography(described):
+            kept = _geography(described)
+            self._left_out |= {r["id"] for r, _ in described} - {f.id for _, f in kept}
+            for raw, field in _of_the_country(kept):
                 self._kinds[field.id], self._pages[field.id] = raw["kind"], number
                 fields.append(field)
         self._fields = {f.id: f for f in fields}
         return fields
 
+    def revealed(self, page: Page) -> list[Field]:
+        """The fields of every page, once filled: an address row follows the country, which no
+        read lists but at a request per country, and a question can follow an answer."""
+        _settle(page)
+        for number in range(1, max(len(_page_numbers(page)), 1) + 1):
+            _go_to(page, number)
+            known = {*self._fields, *self._left_out}
+            described = [
+                (raw, field)
+                for raw in page.evaluate(_READ)
+                if raw["id"] not in known and (field := self._described(page, raw))
+            ]
+            for raw, field in _geography(described):
+                self._kinds[field.id], self._pages[field.id] = raw["kind"], number
+                self._fields[field.id] = field
+        return list(self._fields.values())
+
     def fill(self, page: Page, answers: dict[str, Answer]) -> None:
+        """Passes the email step under the answer's address and fills the form; once it did, what
+        `revealed` found, on the form as it stands."""
+        if not self._filling:
+            self._start(page, answers)
+            self._filling = True
+        self._fill_pages(page, answers)
+
+    def _start(self, page: Page, answers: dict[str, Answer]) -> None:
         if LEGAL in self._fields and answers.get(LEGAL) != "Yes":
             raise Unsupported("not_qualified: the form opens only once the terms are accepted")
         page.context.clear_cookies()
@@ -309,6 +341,8 @@ class Oracle:
         email = answers.get(EMAIL)
         assert isinstance(email, str)  # required, so answered
         self._pass_email_step(page, email, LEGAL in self._fields)
+
+    def _fill_pages(self, page: Page, answers: dict[str, Answer]) -> None:
         for number in sorted(set(self._pages.values())):
             on_page = [i for i in answers if self._pages.get(i) == number]
             if not on_page:
@@ -414,7 +448,7 @@ class Oracle:
         """The innermost row holding the field."""
         selector = page.evaluate(_ROW, field_id)
         if selector is None:
-            raise RuntimeError(f"{field_id} left the form while it was filled")
+            raise NotTaken(f"{field_id} is not on the form under the answers given before it")
         return page.locator(".input-row").filter(has=page.locator(selector)).last
 
     def _options(self, page: Page, field_id: str) -> list[str]:
@@ -560,6 +594,24 @@ def _geography(described: list[tuple[dict, Field]]) -> list[tuple[dict, Field]]:
     ]
 
 
+def _of_the_country(described: list[tuple[dict, Field]]) -> list[tuple[dict, Field]]:
+    """Keys the address rows to the page's country: required under it alone, as another has rows
+    of its own, which the fill reveals, and a list's items its alone, another's a request each."""
+    country = next((f.default for _, f in described if f.id == COUNTRY), None)
+    if not isinstance(country, str):
+        return described
+    held = Is(COUNTRY, [country])
+
+    def keyed(field: Field) -> Field:
+        if field.required:
+            field = replace(field, required=False, required_when=held)
+        if field.options:
+            field = replace(field, options=None, option_sets=[OptionSet(held, field.options)])
+        return field
+
+    return [(r, keyed(f) if r.get("address") else f) for r, f in described]
+
+
 def _page_numbers(page: Page) -> list[str]:
     """The page numbers of a form over several pages, none for a single page."""
     return [_text(t) for t in page.locator(PAGE_NUMBERS).all_inner_texts() if _text(t).isdigit()]
@@ -617,6 +669,18 @@ def _open_list(page: Page, box: Locator) -> list[str]:
     return options
 
 
+def _list_settled(page: Page, cells: Locator) -> list[str]:
+    """A list's items, once the search typed has stopped narrowing it."""
+    items = None
+    for _ in range(LIST_POLLS):
+        now = [_text(t) for t in cells.all_inner_texts()]
+        if now == items:
+            break
+        items = now
+        page.wait_for_timeout(LIST_POLL_MS)
+    return items or []
+
+
 def _choose(
     page: Page,
     box: Locator,
@@ -637,15 +701,15 @@ def _choose(
         cells = page.locator(f"[id='{box.get_attribute('aria-controls')}'] [role=gridcell]")
         with suppress(browser.Timeout):
             cells.first.wait_for(timeout=SEARCH_TIMEOUT_MS)
-        for i in range(cells.count()):
-            item = _text(cells.nth(i).inner_text())
+        for item in _list_settled(page, cells):
             if (
                 (exact and item == answer)
                 or (code and re.search(rf"{re.escape(answer)}(\D|$)", item))
                 or (field is not None and matching.matches(field, answer, item))
             ):
-                cells.nth(i).scroll_into_view_if_needed()
-                _click(page, cells.nth(i))
+                cell = cells.filter(has_text=_exact(item)).first  # by its text: a late search
+                cell.scroll_into_view_if_needed()  # moves it, or leaves it alone in the list
+                _click(page, cell)
                 return
         box.press("Escape")
         if field is not None:

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/paycor.py

@@ -268,6 +268,10 @@ class Paycor:
                         f"unhandled_rendering: cannot fill {field.id}, a {kind}, with {answer!r}"
                     )
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The form's controls; the answers to the pages before it are hidden inputs by now."""
+        return [c["id"] for c in page.evaluate(_CONTROLS)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         held: dict[str, Answer] = {}
         for field, step in self._before:

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/personio.py

@@ -111,6 +111,10 @@ class Personio:
                 case _:
                     control.fill(value)
 
+    def shown_fields(self, page: Page) -> list[str]:
+        form = html.parse(page.locator(FORM).evaluate("f => f.outerHTML"))
+        return [f.id for f, _ in _form(form)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, [f.id for f in self._fields])
         out: dict[str, Answer] = {}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/recruitee.py

@@ -149,6 +149,10 @@ class Recruitee:
                 case _:
                     controls.first.fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        form = html.parse(page.locator(FORM).evaluate("f => f.outerHTML"))
+        return [raw["id"] for raw in _read(form)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, self._kinds)
         return {k: v for k, v in values.items() if v not in (None, "", [])}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/rippling.py

@@ -159,6 +159,26 @@ _SHOWN = """(root, [id, nth, testids]) => {
   return {kind: "unknown"};
 }"""
 
+# The fields the page shows, one `[data-testid=field]` block each, named by the first
+# `data-testid` in it that names a field. A second block of one id is its "Other" text box.
+_FIELDS = """(root, names) => {
+  const out = new Set();
+  const control = "input:not([type=hidden]), textarea, [role=combobox], [role=radio], "
+    + "[role=checkbox]";
+  const generic = ["field", "screen-reader-only"];
+  for (const block of root.querySelectorAll("[data-testid=field]")) {
+    if (block.parentElement.closest("[data-testid=field]")) continue;
+    if (!block.getClientRects().length || !block.querySelector(control)) continue;
+    const el = [...block.querySelectorAll("[data-testid]")]
+      .find(e => !generic.includes(e.dataset.testid));
+    if (!el) continue;
+    const id = el.dataset.testid.replace(/_others$/, "_other").replace(/-code$/, "");
+    const name = names[id] ?? id;
+    out.add(out.has(name) ? `${name}_other` : name);
+  }
+  return [...out];
+}"""
+
 
 class Rippling:
     name = "rippling"
@@ -236,9 +256,7 @@ class Rippling:
             if field_id.endswith("_comment"):
                 _comment(page, field_id.removesuffix("_comment"), value)
             elif field_id.endswith("_other"):
-                page.locator(f"[data-testid='{field_id.removesuffix('_other')}']").nth(1).locator(
-                    "input"
-                ).fill(value)
+                _other(page, field_id.removesuffix("_other")).fill(value)
             elif field_id == "phone_number":
                 _phone(page, value)
             elif field_id == "location":
@@ -248,11 +266,15 @@ class Rippling:
             else:
                 _fill(page, field_id, answer)
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return page.locator("body").evaluate(_FIELDS, {v: k for k, v in TESTIDS.items()})
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         body, out = page.locator("body"), {}
         for field_id in self._fields:
             if field_id.endswith("_other"):
-                shown = body.evaluate(_SHOWN, [field_id.removesuffix("_other"), 1, TESTIDS])
+                box = _other(page, field_id.removesuffix("_other"))
+                shown = {"value": box.input_value() if box.count() else None}
             elif field_id == "phone_number":
                 code = page.locator("[data-testid='phone_number-code'] input").input_value()
                 number = page.locator("[data-testid='input-phone_number']").input_value()
@@ -358,6 +380,13 @@ def _box(page: Page, field_id: str) -> Locator:
     return page.locator(f"[data-testid='{TESTIDS.get(field_id, field_id)}']").first
 
 
+def _other(page: Page, field_id: str) -> Locator:
+    """The text box an "Other" answer shows: `<id>_others` under boxes, a second `<id>` under
+    radios."""
+    second = page.locator(f"[data-testid='{field_id}']").nth(1)
+    return page.locator(f"[data-testid='{field_id}_others']").or_(second).locator("input")
+
+
 def _option(page: Page, field_id: str, value: str) -> None:
     """Picks the open list's option whose text is `value`."""
     option = page.locator("[role=listbox] [role=option]").filter(has_text=_exact(value))

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/successfactors.py

@@ -439,6 +439,9 @@ class SuccessFactors:
             raise NotTaken(f"{key} is no longer on the page")
         return found | {"options": self._fields[key]["options"]}
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [d["key"] for d in page.evaluate(FIELDS)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         """Each field's value; an attachment's, the file uploaded to it when it lists several."""
         values = {}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/ukg.py

@@ -35,7 +35,6 @@ SETTLE_MS = 500
 YES_NO = ["Yes", "No"]
 DATE_SHOWN = re.compile(r"(\d\d)/(\d\d)/(\d{4})")
 COUNTRY, STATE = "Country", "State"
-HISPANIC = "HispanicOrigin"  # race (`EthnicOrigin`) is asked when not Hispanic or Latino
 DECLINE_BOX = "input[type=checkbox][data-automation$='-decline-checkbox']"
 DAYS, DOCUMENTS = "availability_days", "documents"
 
@@ -50,7 +49,9 @@ ERRORS = [
 ]  # fmt: skip
 
 # Each visible field of the form, in page order: a control's id (a radio group's name), its
-# label, how it renders, whether it is required, its options and value.
+# label, how it renders, whether it is required, its options and value. Two follow-ups are
+# listed hidden, `unshown`, after the answer that shows them: race, when not Hispanic or Latino,
+# and the referrer's details, when referred by an employee.
 _READ = """root => {
   const clean = s => (s || "").replace(/\\s+/g, " ").trim();
   const shown = el => !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length) &&
@@ -87,6 +88,7 @@ _READ = """root => {
     const type = (el.type || "").toLowerCase();
     if (["hidden", "file", "submit", "button"].includes(type) || !shown(el)) continue;
     if (el.matches("[data-automation$='-decline-checkbox']")) continue;  // its select's option
+    if (el.matches("[data-automation=select-document-checkbox]")) continue;  // the upload's own
     if (el.closest("ukg-input, ukg-input-container, [data-automation=upload-resume]")) continue;
     const id = type === "radio" ? el.name : el.id || el.name || el.dataset.automation;
     if (!id || seen.has(id)) continue;
@@ -131,8 +133,25 @@ _READ = """root => {
     const label = labelOf(race);
     out.splice(out.findIndex(f => f.id === "HispanicOrigin") + 1, 0, select(race, {
       id: race.id, label: clean(label?.textContent), required: required(race, label),
-      readonly: false, maxLength: null,
-      shownWhen: clean([...hispanic.options].find(o => o.value === "No")?.textContent)}));
+      readonly: false, maxLength: null, unshown: !shown(race),
+      shownWhen: ["HispanicOrigin",
+                  clean([...hispanic.options].find(o => o.value === "No")?.textContent)]}));
+  }
+  const referred = root.querySelector("input[type=radio][name=employeereferral][value=true]");
+  if (referred && out.some(f => f.id === "employeereferral")) {
+    const details = ["ReferralName", "ReferralEmail", "ReferralPhone"]
+      .map(id => root.querySelector(`input[id="${id}"]`)).filter(Boolean);
+    for (const el of details) {
+      const kept = out.findIndex(f => f.id === el.id);
+      if (kept >= 0) out.splice(kept, 1);
+    }
+    out.splice(out.findIndex(f => f.id === "employeereferral") + 1, 0, ...details.map(el => {
+      const label = labelOf(el);
+      return {id: el.id, label: clean(label?.textContent), required: required(el, label),
+              kind: "text", options: [], value: "", readonly: false, maxLength: el.maxLength,
+              unshown: !shown(el),
+              shownWhen: ["employeereferral", clean(referred.closest("label").textContent)]};
+    }));
   }
   const days = [...root.querySelectorAll("[data-automation=day-btn]")].filter(shown);
   if (days.length) {
@@ -396,6 +415,9 @@ class UKG:
                 case _:
                     _control(root, field_id).fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ) if not raw.get("unshown")]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, self._kinds)
         return {k: v for k, v in values.items() if v not in (None, "", [])}
@@ -496,7 +518,8 @@ def _field(raw: dict[str, Any], states: dict[str, list[str]] | None) -> Field:
         return Field(field_id, label, KINDS[kind], raw["required"], prefilled=value,
                      unfillable=reason)  # fmt: skip
     options = raw["options"] if kind in ("select", "radio") else None
-    shown_when = Is(HISPANIC, [raw["shownWhen"]]) if raw.get("shownWhen") else None
+    when = raw.get("shownWhen")
+    shown_when = Is(when[0], [when[1]]) if when else None
     if field_id == STATE and states is not None:
         sets = [OptionSet(Is(COUNTRY, [country]), names) for country, names in states.items()]
         return Field(field_id, label, FieldType.CHOICE, raw["required"], [], option_sets=sets)

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/welcomekit.py

@@ -179,6 +179,11 @@ class WelcomeKit:
                 case _:
                     control.fill(value)
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The older form's controls; the newer form's API lists every field whatever the
+        answers, so it has nothing to tell."""
+        return [] if self._new else [c.field.id for c in _old_controls(page.content())]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         form = page.locator("form").first if self._new else page.locator(OLD)
         controls = [

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/wellfound.py

@@ -49,6 +49,8 @@ DECLINE = "Skip for now"  # its other button, Start my application, enrols the a
 FORM = 'form:has([data-test="JobApplicationModal--SubmitButton"])'
 SEND = '[data-test="JobApplicationModal--SubmitButton"]'
 NOTE = "note"
+NOTE_INPUT = "userNote"
+QUESTION = re.compile(r"customQuestionAnswers\[(\d+)\]")  # the start of a question's control name
 GONE = "no longer available"
 WRONG_PASSWORD = "Email or password is incorrect"
 PASSWORD_LENGTH = (8, 128)
@@ -258,6 +260,12 @@ class Wellfound:
                     said = f"cannot fill {field_id}, a {kind}, with {answer!r}"
                     raise Unsupported(f"unhandled_rendering: {said}")
 
+    def shown_fields(self, page: Page) -> list[str]:
+        """The dialog's controls, named as the read names its questions and its note."""
+        names = page.locator(FORM).evaluate("""form => [...form.querySelectorAll("[name]")]
+          .filter(e => e.getClientRects().length).map(e => e.name)""")
+        return list(dict.fromkeys(i for name in names if (i := _field_named(name))))
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         form = page.locator(FORM)
         values: dict[str, Answer] = {}
@@ -373,6 +381,14 @@ def _opened(page: Page, read: list[dict[str, Any]]) -> dict[str, Any]:
     raise RuntimeError("Apply opened no dialog")
 
 
+def _field_named(name: str) -> str | None:
+    """The field a control of the dialog belongs to."""
+    if name == NOTE_INPUT:
+        return NOTE
+    question = QUESTION.match(name)
+    return f"question_{question[1]}" if question else None
+
+
 def _field(question: dict[str, Any]) -> Field:
     field_id, label = f"question_{question['id']}", question["question"].strip()
     required, options = question["required"], [o["option"] for o in question["options"]]

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/werecruit.py

@@ -122,6 +122,9 @@ class WeRecruit:
                 case _:
                     control.fill(value)
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [c.field.id for c in _parse(page.content()) or []]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         shown = [c for c in self._controls if not c.field.unfillable]
         values = page.locator(FORM).evaluate(_READ_BACK, [c.name for c in shown])

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/workable.py

@@ -243,6 +243,9 @@ class Workable:
                 case _:
                     field.fill(values[0])
 
+    def shown_fields(self, page: Page) -> list[str]:
+        return [raw["id"] for raw in page.locator(FORM).evaluate(_READ)]
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).evaluate(_READ_BACK, self._kinds)
         for field_id, fmt in self._formats.items():

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/workatastartup.py

@@ -97,6 +97,20 @@ _READ_BACK = """dialog => {
   return out;
 }"""
 
+# The fields the dialog shows, once tagged; a control `_TAG` did not place, by its place in the
+# dialog. The relocation box is the account profile's, not a field.
+_SHOWN = """dialog => {
+  const out = new Set();
+  [...dialog.querySelectorAll("input, select, textarea")].forEach((el, n) => {
+    if (el.type === "hidden" || !el.getClientRects().length) return;
+    const block = el.closest("[data-atsfill-question]");
+    if (block) out.add("question_" + block.dataset.atsfillQuestion);
+    else if (el.dataset.atsfillField === "message") out.add("message");
+    else if (!el.dataset.atsfillField) out.add(`control #${n + 1}`);
+  });
+  return [...out];
+}"""
+
 
 class WorkAtAStartup:
     name = "workatastartup"
@@ -261,6 +275,10 @@ class WorkAtAStartup:
         if relocation.count():
             relocation.uncheck()
 
+    def shown_fields(self, page: Page) -> list[str]:
+        self._tag(page)
+        return page.locator(DIALOG).first.evaluate(_SHOWN)
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         self._tag(page)
         values = page.locator(DIALOG).first.evaluate(_READ_BACK)

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/workday.py

@@ -322,8 +322,10 @@ class Workday:
             _add_entries(page, answers)
             filled: set[str] = set()
             for _ in range(FOLLOW_UP_ROUNDS):  # an answer shows its follow-up questions
-                todo = [r for r in _rendered(page, answers) if r["id"] not in filled]
-                todo = [r for r in todo if r["id"] in answers and r["id"] in self._fields]
+                todo = [r for r in _rendered(page, answers) if r["id"] in answers]
+                todo = [r for r in todo if r["id"] in self._fields]
+                # a reply that came after its wait rendered over the answers given meanwhile
+                todo = [r for r in todo if r["id"] not in filled or _lost(r, answers[r["id"]])]
                 if not todo:
                     break
                 for raw in todo:
@@ -560,6 +562,11 @@ def _filled(page: Page, answers: dict[str, Answer]) -> list[dict[str, Any]]:
     return raws
 
 
+def _lost(raw: dict[str, Any], answer: Answer) -> bool:
+    """Whether the page dropped a field's answer; an upload still attaching shows no value."""
+    return bool(answer) and raw["kind"] != "file" and raw["value"] in (None, "", [])
+
+
 def _field(raw: dict[str, Any]) -> Field | None:
     kind, options = KINDS.get(raw["kind"]), raw["options"]
     if raw["kind"] == "box":

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/adapters/wttj.py

@@ -36,9 +36,10 @@ SUGGEST_POLL_MS = 250
 KEY_DELAY_MS = 30
 YES_NO = ["Yes", "No"]
 
-# The fillable fields: an input's name is its id. Email comes from the account, disabled.
+# The fillable fields: an input's name is its id. Email comes from the account, disabled; an
+# upload shows `save_resume`, whether the CV stays on the account, left as it is.
 _READ = """form => [...form.querySelectorAll("input[name], textarea[name], select[name]")]
-  .filter(e => !e.disabled && e.name !== "avatar")
+  .filter(e => !e.disabled && !["avatar", "save_resume"].includes(e.name))
   .map(e => {
     const label = form.querySelector(`label[for="${CSS.escape(e.id)}"]`);
     return {id: e.name, tag: e.tagName.toLowerCase(), type: e.type,
@@ -172,6 +173,10 @@ class WelcomeToTheJungle:
                     )
                     raise Unsupported(reason)
 
+    def shown_fields(self, page: Page) -> list[str]:
+        raws = page.locator(FORM).first.evaluate(_READ)
+        return list(dict.fromkeys(r["id"] for r in raws if r["type"] != "hidden"))
+
     def read_back(self, page: Page) -> dict[str, Answer]:
         values = page.locator(FORM).first.evaluate(_READ_BACK, list(self._fields))
         return {k: v for k, v in values.items() if v not in (None, "")}

{atsfill-0.0.7.dev211 → atsfill-0.0.8.dev219}/src/atsfill/core/browser.py

@@ -18,10 +18,10 @@ from urllib.parse import urljoin
 from urllib.request import Request as HttpRequest
 from urllib.request import urlopen
 
-from camoufox.addons import ADDONS_DIR, DefaultAddons, get_addon_path, maybe_download_addons
+from camoufox.addons import DefaultAddons, get_addon_path
 from camoufox.exceptions import CamoufoxNotInstalled
 from camoufox.fingerprints import Screen, generate_fingerprint
-from camoufox.pkgman import installed_verstr
+from camoufox.pkgman import INSTALL_DIR, installed_verstr, launch_path
 from camoufox.sync_api import Camoufox
 from filelock import FileLock
 from filelock import Timeout as LockTimeout
@@ -58,11 +58,9 @@ BUDGET = 180.0  # seconds per command, lock wait included
 PROFILE_BUDGET = 300.0  # seconds for a `profile` run, which may save every profile step
 LOCK_POLL_S = 0.1
 HTTP_TIMEOUT_S = 20  # per request to a hiring system's API
-BUILD = "156.0.1-beta.36"  # the Camoufox browser build, pinned so results compare across machines
-BLOCKER = ADDONS_DIR / "UBO-frozen"  # Camoufox's uBlock Origin: annoyance lists on, updates off
+BUILD = "158.0.1-beta.41"  # the Camoufox browser build, pinned so results compare across machines
+BLOCKER = INSTALL_DIR / "atsfill" / f"UBO-{BUILD}"  # the build's uBlock Origin, annoyance lists on
 BLOCKER_LISTS = "annoyances"  # the group of uBlock Origin lists: cookie banners, pop-ups, widgets
-# On, its first update, 37s into a launch, leaves the blocker hiding nothing for the rest of it.
-BLOCKER_AUTO_UPDATE = "autoUpdate: true,"
 # The adapters' filters for the banners the lists miss: a list of the blocker's, on like them.
 FILTERS = "assets/atsfill/filters.txt"
 INCLUDE = re.compile(r"^!#include (\S+)$", re.M)
@@ -90,14 +88,13 @@ def version() -> str:
 
 
 def install() -> int:
-    """Fetches the pinned browser build unless it is installed, then the add-ons every launch
-    loads, so that browsers started side by side do not each download them."""
+    """Fetches the pinned browser build unless it is installed, then builds the blocker every
+    launch loads, so that browsers started side by side do not each build it."""
     if version() != BUILD:
-        subprocess.call([sys.executable, "-m", "camoufox", "fetch", f"official/{BUILD}"])
+        subprocess.call([sys.executable, "-m", "camoufox", "fetch"])  # the package's paired build
     if version() != BUILD:  # `camoufox fetch` exits 0 when it fails
         print(f"the Camoufox build {BUILD} is not installed", file=sys.stderr)
         return 1
-    maybe_download_addons(list(DefaultAddons))
     try:
         _build_blocker()
     except OSError as e:
@@ -109,13 +106,13 @@ def install() -> int:
 def _build_blocker() -> None:
     """Copies Camoufox's uBlock Origin to `BLOCKER` with only its annoyance lists on, read
     from disk: fetched by each launch, they would hold its first page for seconds, and each
-    list on is compiled by each launch before the blocker starts. Its updates stay off: a fresh
-    profile would drop them with the browser anyway."""
-    ADDONS_DIR.mkdir(parents=True, exist_ok=True)
-    with FileLock(ADDONS_DIR / f"{BLOCKER.name}.lock"):
+    list on is compiled by each launch before the blocker starts. Camoufox's policy keeps it
+    from updating them: its first update, 37s into a launch, left it hiding nothing."""
+    BLOCKER.parent.mkdir(parents=True, exist_ok=True)
+    with FileLock(BLOCKER.with_name(f"{BLOCKER.name}.lock")):
         build = BLOCKER.with_name(f"{BLOCKER.name}.build")
         shutil.rmtree(build, ignore_errors=True)
-        shutil.copytree(get_addon_path(DefaultAddons.UBO.name), build)
+        shutil.copytree(get_addon_path(DefaultAddons.UBO.name, launch_path()), build)
         assets = build / "assets" / "assets.json"
         entries = json.loads(assets.read_text())
         for key, entry in entries.items():
@@ -135,10 +132,6 @@ def _build_blocker() -> None:
         (build / FILTERS).parent.mkdir()
         (build / FILTERS).touch()
         assets.write_text(json.dumps(entries))
-        settings = build / "js" / "background.js"
-        defaults = settings.read_text()
-        assert BLOCKER_AUTO_UPDATE in defaults  # gone from a new uBlock Origin: find its new form
-        settings.write_text(defaults.replace(BLOCKER_AUTO_UPDATE, "autoUpdate: false,"))
         shutil.rmtree(BLOCKER, ignore_errors=True)
         build.rename(BLOCKER)
 
@@ -193,7 +186,6 @@ def lock(data_dir: Path, deadline: float) -> Generator[None]:
 @contextmanager
 def _launch(data_dir: Path, headless: bool) -> Generator[Browser]:
     if not (BLOCKER / FILTERS).exists():  # installed before the blocker, or its filters, were
-        maybe_download_addons(list(DefaultAddons))
         _build_blocker()
     _write_filters()
     with Camoufox(