atsfill 0.0.5.dev180__tar.gz → 0.0.5.dev182__tar.gz

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atsfill
-Version: 0.0.5.dev180
+Version: 0.0.5.dev182
 Summary: Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle.
 Keywords: job-applications,ats,ashby,greenhouse,teamtailor,ai-agents,cli
 Author: shbernal

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.5.dev180"
+version = "0.0.5.dev182"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.5.dev180"
+version = "0.0.5.dev182"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/ashby.py

@@ -78,7 +78,9 @@ class Ashby:
 
     def __init__(self) -> None:
         self._forms: list[dict[str, Any]] = []  # the application form, then each survey
-        self._pending: set[Any] = set()  # GraphQL requests in flight, since `browser_read`
+        self._pending: set[Any] = set()  # GraphQL requests in flight, since `open_form`
+        self._listening = False
+        self._loaded: Response | None = None  # the ApiJobPosting response `open_form` caught
         self._last_change = 0.0
         self._hosted: str | None = None  # the jobs.ashbyhq.com posting a custom domain embeds
 
@@ -125,14 +127,24 @@ class Ashby:
         except OSError, ValueError, KeyError, TypeError, Unsupported:
             return None
 
-    def browser_read(self, page: Page) -> list[Field]:
-        page.on("request", self._on_request)
-        page.on("requestfinished", self._on_finished)
-        page.on("requestfailed", self._on_finished)
-        page.on("response", self._on_response)
+    def open_form(self, page: Page, url: str) -> None:
+        """Loads `/application`, listening first for the page's own ApiJobPosting response."""
+        if not self._listening:
+            page.on("request", self._on_request)
+            page.on("requestfinished", self._on_finished)
+            page.on("requestfailed", self._on_finished)
+            page.on("response", self._on_response)
+            self._listening = True
         with page.expect_response(lambda r: _op(r) == "ApiJobPosting") as posting:
-            page.goto(self.posting(page.url) + "/application")
-        job = posting.value.json()["data"]["jobPosting"]
+            page.goto(self.posting(url) + "/application")
+        self._loaded = posting.value
+
+    def browser_read(self, page: Page) -> list[Field]:
+        if self._loaded is None:  # a page claimed once loaded: an embed
+            self.open_form(page, page.url)
+        assert self._loaded
+        loaded, self._loaded = self._loaded, None
+        job = loaded.json()["data"]["jobPosting"]
         if job is None:
             raise Unsupported("posting_gone")
         self._forms = [job["applicationForm"], *job["surveyForms"]]

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/breezy.py

@@ -211,6 +211,10 @@ _FILE_SHOWN = """e => {
 }"""
 
 
+def _on_form(url: str) -> bool:
+    return urlsplit(url).path.rstrip("/").endswith("/apply")
+
+
 class Breezy:
     name = "breezy"
     known_gaps: dict[str, Gap] = {}
@@ -236,9 +240,12 @@ class Breezy:
         match = JOB.match(parts.path)
         return f"https://{parts.hostname}/p/{match[1]}" if match else url
 
+    def open_form(self, page: Page, url: str) -> None:
+        page.goto(url if _on_form(url) else f"{self.posting(url)}/apply")
+
     def browser_read(self, page: Page) -> list[Field]:
-        if not urlsplit(page.url).path.rstrip("/").endswith("/apply"):
-            page.goto(f"{self.posting(page.url)}/apply")
+        if not _on_form(page.url):
+            self.open_form(page, page.url)
         form = page.locator(FORM)
         try:
             form.locator("[name=cName]").wait_for(state="attached", timeout=FORM_TIMEOUT_MS)

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/lever.py

@@ -116,6 +116,10 @@ _READ_BACK = """(form, kinds) => {
 }"""
 
 
+def _on_form(url: str) -> bool:
+    return urlsplit(url).path.rstrip("/").endswith("/apply")
+
+
 class Lever:
     name = "lever"
     known_gaps: dict[str, Gap] = {}
@@ -134,9 +138,12 @@ class Lever:
         parts, job = urlsplit(url), _job(url)
         return f"https://{parts.hostname}/{job[0]}/{job[1]}" if job else url
 
+    def open_form(self, page: Page, url: str) -> None:
+        page.goto(url if _on_form(url) else f"{self.posting(url)}/apply")
+
     def browser_read(self, page: Page) -> list[Field]:
-        if not urlsplit(page.url).path.rstrip("/").endswith("/apply"):
-            page.goto(f"{self.posting(page.url)}/apply")
+        if not _on_form(page.url):
+            self.open_form(page, page.url)
         form = page.locator(FORM)
         try:
             form.wait_for(state="attached", timeout=FORM_TIMEOUT_MS)

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/manatal.py

@@ -12,7 +12,7 @@ from typing import Any
 from urllib.parse import urlsplit
 
 from atsfill.core.adapter import Blocked, Gap, Unsupported, known_gap
-from atsfill.core.browser import HTTP_TIMEOUT_S, Page
+from atsfill.core.browser import HTTP_TIMEOUT_S, Page, Response
 from atsfill.core.results import Answer, Field, FieldType, Is, SideEffect
 
 HOST = "www.careers-page.com"
@@ -183,6 +183,7 @@ class Manatal:
 
     def __init__(self) -> None:
         self._entries: list[dict[str, Any]] = []
+        self._loaded: Response | None = None  # the form's GET `open_form` caught
 
     def detect(self, url: str, page: Page | None) -> bool:
         parts = urlsplit(url).path.strip("/").split("/")
@@ -210,13 +211,20 @@ class Manatal:
         except OSError, ValueError, KeyError, TypeError:
             return None
 
-    def browser_read(self, page: Page) -> list[Field]:
+    def open_form(self, page: Page, url: str) -> None:
+        """Loads `/apply`, listening first for the form's GET."""
         with page.expect_response(
             lambda r: r.request.method == "GET" and "/application-form/" in r.url,
             timeout=LOAD_TIMEOUT_MS,
         ) as loaded:
-            page.goto(_posting(page.url) + "/apply")
-        response = loaded.value
+            page.goto(_posting(url) + "/apply")
+        self._loaded = loaded.value
+
+    def browser_read(self, page: Page) -> list[Field]:
+        if self._loaded is None:
+            self.open_form(page, page.url)
+        assert self._loaded
+        response, self._loaded = self._loaded, None
         if response.headers.get("x-amzn-waf-action") == "challenge":
             raise Blocked("captcha: AWS WAF challenge")
         if not response.ok:

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/phenom.py

@@ -15,6 +15,7 @@ import time
 import urllib.request
 import uuid
 from collections.abc import Callable, Generator, Iterable, Iterator, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager, suppress
 from datetime import date
 from html import unescape
@@ -64,6 +65,7 @@ FILL_ROUNDS = 6  # an answer shows a follow-up, which can show another
 PICK_TRIES = 3
 MAX_STEPS = 12
 OPTIONS_TIMEOUT_S = 8
+LOADERS_AT_ONCE = 8  # a loader request a country: 245 on one site
 YES_NO = ["Yes", "No"]
 NO_YES = ["No", "Yes"]
 MONTH = " (MM/YYYY)"
@@ -107,7 +109,7 @@ ANSWER = re.compile(r"%%formData\.(\w+)%%")
 SEARCHING = re.compile(r"Searching", re.IGNORECASE)  # a typeahead's menu before its results
 ACCEPTED = re.compile(r"\b(DOCX?|PDF|TXT|HTML?|RTF)\b", re.IGNORECASE)
 
-type Post = Callable[[dict[str, Any]], Any]
+type Post = Callable[[list[dict[str, Any]]], list[Any]]  # None for a failed request
 
 # Each field the current step shows: its control's kind, id, label and requirement. A field is
 # a `.form-group.field` with no field inside it; a checkbox group is named by its boxes' id.
@@ -226,14 +228,30 @@ _SHOWS = """text => [...document.querySelectorAll('body *')].some(el =>
 
 # A request from the page, as its own scripts make them: the CDN's configuration, or a post to
 # `/applySubmit` with the session's token.
-_FETCH = """async ([url, body, token]) => {
-  const init = body ? {method: 'POST', body: JSON.stringify(body),
-    headers: {'Content-Type': 'application/json', 'x-csrf-token': token}} : {};
-  const response = await fetch(url, init);
+_FETCH = """async url => {
+  const response = await fetch(url);
   if (!response.ok) throw new Error(`${response.status} from ${url}`);
   return response.json();
 }"""
 
+# Posts each body to `/applySubmit`, `most` at a time; null for one that fails.
+_POSTS = """async ([bodies, token, most]) => {
+  const out = bodies.map(() => null);
+  let next = 0;
+  const post = async () => {
+    for (let i = next++; i < bodies.length; i = next++) {
+      try {
+        const response = await fetch('/applySubmit', {method: 'POST',
+          body: JSON.stringify(bodies[i]),
+          headers: {'Content-Type': 'application/json', 'x-csrf-token': token}});
+        if (response.ok) out[i] = await response.json();
+      } catch {}
+    }
+  };
+  await Promise.all(Array.from({length: most}, post));
+  return out;
+}"""
+
 
 class Phenom:
     name = "phenom"
@@ -292,6 +310,16 @@ class Phenom:
             with opener.open(request, timeout=HTTP_TIMEOUT_S) as response:
                 return response.read()
 
+        def post(body: dict[str, Any]) -> Any:
+            try:
+                return json.loads(load(f"{origin}/applySubmit", body, token))
+            except OSError, ValueError:
+                return None
+
+        def posts(bodies: list[dict[str, Any]]) -> list[Any]:
+            with ThreadPoolExecutor(LOADERS_AT_ONCE) as pool:
+                return list(pool.map(post, bodies))
+
         try:
             if prefix[2] == "job":
                 job = _phapp_of(load(url).decode())
@@ -299,11 +327,7 @@ class Phenom:
                     return None
             ph = _phapp_of(load(f"{origin}{prefix[1]}/apply?jobSeqNo={seq}").decode())
             token = get(ph, "sessionParams.csrfToken") or ""
-            fields, _ = _read(
-                ph,
-                lambda url: json.loads(load(url)),
-                lambda body: json.loads(load(f"{origin}/applySubmit", body, token)),
-            )
+            fields, _ = _read(ph, lambda url: json.loads(load(url)), posts)
             return fields
         except OSError, ValueError, KeyError, TypeError, RuntimeError, Unsupported:
             return None
@@ -320,8 +344,8 @@ class Phenom:
         try:
             fields, raws = _read(
                 ph,
-                lambda url: page.evaluate(_FETCH, [url, None, token]),
-                lambda body: page.evaluate(_FETCH, ["/applySubmit", body, token]),
+                lambda url: page.evaluate(_FETCH, url),
+                lambda bodies: page.evaluate(_POSTS, [bodies, token, LOADERS_AT_ONCE]),
             )
         except (BrowserError, ValueError, KeyError) as e:
             raise RuntimeError(f"no configuration for the wizard ({e})") from None
@@ -653,7 +677,7 @@ class _Config:
         out = [raw]
         if html == "country":
             raw["loads"] = "getCountrySpecificFields" in (get(u, "apiConfig.payload") or "")
-            out += self.country_fields(p, u, path, nav)
+            out += self.country_fields(u, path, nav)
         return out
 
     def options(
@@ -870,18 +894,24 @@ class _Config:
         return out
 
     def call(self, payload: str, **extra: Any) -> Any:
-        """Posts a loader's payload to `/applySubmit`, the page's values put in; None when it
-        names an answer, or the request fails."""
+        return self.calls(payload, [extra])[0]
+
+    def calls(self, payload: str, extras: list[dict[str, Any]]) -> list[Any]:
+        """Posts a loader's payload to `/applySubmit` with each of `extras`, the page's values
+        put in; None where it names an answer, or the request fails."""
         if ANSWER.search(payload):
-            return None
+            return [None] * len(extras)
         filled = PAGE_VALUE.sub(lambda m: _json_text(get({"phApp": self.ph}, m[1])), payload)
+        bodies = []
         try:
-            body = json.loads(filled)
-            body.setdefault("formData", {}).update(extra)
-            body["formData"].setdefault("applyAtsSource", self.ddo.get("applyATS"))
-            return self.post(body)
+            for extra in extras:
+                body = json.loads(filled)
+                body.setdefault("formData", {}).update(extra)
+                body["formData"].setdefault("applyAtsSource", self.ddo.get("applyATS"))
+                bodies.append(body)
+            return self.post(bodies)
         except Exception:
-            return None
+            return [None] * len(extras)
 
     def loaded(self, u: dict, nav: Any) -> list[dict[str, Any]]:
         """A part of the form its loader serves: a questionnaire of the job's, where the job
@@ -945,19 +975,26 @@ class _Config:
         }
         return raw
 
-    def country_fields(self, p: dict, u: dict, path: str, nav: Any) -> list[dict[str, Any]]:
-        """The fields the country's answer before any adds under `cntryFields` (names,
-        address)."""
+    def country_fields(self, u: dict, path: str, nav: Any) -> list[dict[str, Any]]:
+        """The fields each country adds under `cntryFields` (names, address), a request a
+        country, in the order of the country's answer before any."""
         payload = (u.get("apiConfig") or {}).get("payload") or ""
-        code, _ = self.prefilled(path)
-        if "getCountrySpecificFields" not in payload or code is None:
+        if "getCountrySpecificFields" not in payload:
             return []
-        part = get(self.call(payload, countryCode=code), "applyGetReferences.data")
-        if not isinstance(get(part, "schema"), dict):
-            return []
-        return self.object(
-            part["schema"], part.get("uiSchema") or {}, "cntryFields.", nav, False, None
-        )
+        values, texts = self.listed.get(path) or ([], [])
+        held, _ = self.prefilled(path)
+        codes = sorted(values, key=lambda c: c != held)
+        parts = self.calls(payload, [{"countryCode": c} for c in codes])
+        by_id: dict[str, dict[Any, dict[str, Any]]] = {}
+        for code, part in zip(codes, parts, strict=True):
+            part = get(part, "applyGetReferences.data")
+            if not isinstance(get(part, "schema"), dict):
+                continue
+            ui = part.get("uiSchema") or {}
+            for raw in self.object(part["schema"], ui, "cntryFields.", nav, False, None):
+                by_id.setdefault(raw["id"], {})[code] = raw
+        shown = dict(zip(values, texts, strict=False))
+        return [_by_country(path, rows, codes, shown) for rows in by_id.values()]
 
     def date_range(
         self, p: dict, u: dict, path: str, nav: Any, when: Condition | None
@@ -1037,6 +1074,52 @@ class _Config:
                 return None if joined is True else joined
 
 
+def _by_country(
+    path: str, rows: dict[Any, dict[str, Any]], codes: list, shown: dict[Any, str]
+) -> dict[str, Any]:
+    """One field from its raw form under each country that lists it: shown, required and
+    listing its options under the countries whose fields say so."""
+    raw = dict(next(iter(rows.values())))
+    raw["when"] = _under(path, {c: r["when"] or True for c, r in rows.items()}, codes)
+    required = _under(
+        path,
+        {c: True if r["required"] else r.get("required_when") or False for c, r in rows.items()},
+        list(rows),
+    )
+    raw["required"] = required is True
+    raw["required_when"] = None if isinstance(required, bool) else required
+    lists = {c: r["options"] for c, r in rows.items() if r["options"]}
+    if lists:
+        raw["kind"] = next(rows[c]["kind"] for c in lists)
+    if lists and (len(lists) < len(rows) or len({tuple(o) for o in lists.values()}) > 1):
+        groups: dict[tuple[str, ...], list[str]] = {}
+        for code, options in lists.items():
+            groups.setdefault(tuple(options), []).append(shown.get(code, str(code)))
+        raw["options"], raw["parent"] = None, path
+        raw["option_sets"] = [OptionSet(Is(path, names), list(o)) for o, names in groups.items()]
+    return raw
+
+
+def _under(path: str, by_value: dict[Any, bool | Condition], values: list) -> bool | Condition:
+    """What holds under each of a list's `values`, as one condition on its answer, False under
+    a value `by_value` leaves out. A part that holds under most values names the others."""
+    groups: dict[str, tuple[bool | Condition, list]] = {}
+    for value, holds in by_value.items():
+        if holds is not False:
+            groups.setdefault(repr(holds), (holds, []))[1].append(value)
+    if len(groups) == 1 and len(next(iter(groups.values()))[1]) == len(values):
+        return next(iter(groups.values()))[0]
+    parts: list[Condition] = []
+    for holds, under in groups.values():
+        on = (
+            Is(path, under)
+            if 2 * len(under) <= len(values)
+            else Is(path, [v for v in values if v not in under], negated=True)
+        )
+        parts.append(AllOf([on, holds]) if not isinstance(holds, bool) else on)
+    return _join(parts, any)
+
+
 def _join(
     parts: Sequence[bool | Condition | None], kind: Callable[[Iterable[object]], bool]
 ) -> bool | Condition:

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/recruitee.py

@@ -62,6 +62,10 @@ _READ_BACK = """(form, kinds) => {
 }"""
 
 
+def _on_form(url: str) -> bool:
+    return urlsplit(url).path.rstrip("/").endswith("/c/new")
+
+
 class Recruitee:
     name = "recruitee"
     known_gaps: dict[str, Gap] = {}
@@ -84,9 +88,12 @@ class Recruitee:
         match = JOB.match(parts.path)
         return f"https://{parts.hostname}/o/{match[1]}" if match else url
 
+    def open_form(self, page: Page, url: str) -> None:
+        page.goto(url if _on_form(url) else f"{self.posting(url)}/c/new")
+
     def browser_read(self, page: Page) -> list[Field]:
-        if not urlsplit(page.url).path.rstrip("/").endswith("/c/new"):
-            page.goto(f"{self.posting(page.url)}/c/new")
+        if not _on_form(page.url):
+            self.open_form(page, page.url)
         form = page.locator(FORM)
         try:
             form.wait_for(state="attached", timeout=FORM_TIMEOUT_MS)

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/teamtailor.py

@@ -125,6 +125,7 @@ class Teamtailor:
         self._follow_ups: set[str] = set()
         self._uploads: dict[str, str] = {}  # file field id: the answer's file name
         self._videos: dict[str, tuple[str, str]] = {}  # video field id: its video id, its URL
+        self._form: str | None = None  # the form `open_form` loaded
 
     def detect(self, url: str, page: Page | None) -> bool:
         """A `*.teamtailor.com` job, or a job page with Teamtailor's "powered by" link or its
@@ -141,13 +142,21 @@ class Teamtailor:
         assert match, url
         return f"https://{parts.hostname}/jobs/{match[1]}"
 
-    def browser_read(self, page: Page) -> list[Field]:
+    def open_form(self, page: Page, url: str) -> None:
         """Opens the form's own page, never the overlay nor "Apply with LinkedIn"."""
-        parts = urlsplit(page.url)
+        parts = urlsplit(url)
         job = JOB.search(parts.path)
-        assert job, page.url  # detected on a job URL
-        page.goto(f"{parts.scheme}://{parts.netloc}{parts.path[: job.end()]}/applications/new")
-        if urlsplit(page.url).hostname != parts.hostname:
+        assert job, url  # detected on a job URL
+        form = f"{parts.scheme}://{parts.netloc}{parts.path[: job.end()]}/applications/new"
+        page.goto(form)
+        self._form = form
+
+    def browser_read(self, page: Page) -> list[Field]:
+        if self._form is None:  # a page claimed once loaded: a custom domain
+            self.open_form(page, page.url)
+        assert self._form
+        opened, self._form = self._form, None
+        if urlsplit(page.url).hostname != urlsplit(opened).hostname:
             raise Unsupported(f"external_step: the job applies on another site ({page.url})")
         form = page.locator(FORM)
         try:

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/adapters/workable.py

@@ -141,6 +141,10 @@ _READ_BACK = """(form, kinds) => {
 }"""
 
 
+def _on_form(url: str) -> bool:
+    return "/apply" in urlsplit(url).path
+
+
 class Workable:
     name = "workable"
     known_gaps = {"entries": Gap.RENDERING}
@@ -174,9 +178,12 @@ class Workable:
         except OSError, ValueError, KeyError, TypeError:
             return None
 
+    def open_form(self, page: Page, url: str) -> None:
+        page.goto(url if _on_form(url) else f"{self.posting(url)}/apply/")
+
     def browser_read(self, page: Page) -> list[Field]:
-        if "/apply" not in urlsplit(page.url).path:
-            page.goto(f"{self.posting(page.url)}/apply/")
+        if not _on_form(page.url):
+            self.open_form(page, page.url)
         form = page.locator(FORM)
         try:
             form.wait_for(timeout=FORM_TIMEOUT_MS)

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/core/adapter.py

@@ -280,6 +280,16 @@ class SendsToOpen(Adapter, Protocol):
         ...
 
 
+@runtime_checkable
+class OpensForm(Adapter, Protocol):
+    """An adapter whose form is another page than the posting: the core has it load the form
+    in place of the posting, when it claims the posting by its URL (ADR 0003)."""
+
+    def open_form(self, page: Page, url: str) -> None:
+        """Loads the form of the posting at `url`, so that `browser_read` finds it open."""
+        ...
+
+
 @runtime_checkable
 class ReadsByRequest(Adapter, Protocol):
     """An adapter with a request read: a form's fields listed without a browser (ADR 0011)."""

{atsfill-0.0.5.dev180 → atsfill-0.0.5.dev182}/src/atsfill/core/commands.py

@@ -29,6 +29,7 @@ from atsfill.core.adapter import (
     ListsShownFields,
     NeedsAccount,
     NotTaken,
+    OpensForm,
     ProfileIncomplete,
     ReadsByRequest,
     Refused,
@@ -173,7 +174,7 @@ def _opened(
             session = None
     with open_page(data, session, seconds) as page:
         try:
-            page.goto(url)
+            _load(page, adapter, url)
         except browser.Timeout:
             if adapter is None:
                 raise
@@ -207,6 +208,14 @@ def _opened(
                     browser.save_session(page, session)
 
 
+def _load(page: Page, adapter: object, url: str) -> None:
+    """Loads the posting, or its form where the adapter that claims it opens it (ADR 0003)."""
+    if isinstance(adapter, OpensForm):
+        adapter.open_form(page, url)
+    else:
+        page.goto(url)
+
+
 _again: ContextVar[Callable[[], Result | None] | None] = ContextVar("again", default=None)
 
 
@@ -281,7 +290,7 @@ def _sign_in_again(
             return _login_required(url, adapter, user)
         session = accounts.session_file(mine, adapter.name, scope)
         accounts.sign_in(page, adapter, scope, *credentials, session)
-        page.goto(url)
+        _load(page, adapter, url)
         return None
     email = credentials[0] if credentials else ""
     scoped = url if adapter.per_employer else None