freemyid 0.2.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. freemyid/__init__.py +2 -0
  2. freemyid/_data/brokers/LICENSE-DATA.md +28 -0
  3. freemyid/_data/brokers/registry.sample.yaml +674 -0
  4. freemyid/_data/brokers/state_laws.yaml +448 -0
  5. freemyid/_data/schema/registry.schema.json +668 -0
  6. freemyid/_data/schema/setup-config.schema.json +538 -0
  7. freemyid/_data/schema/state_laws.schema.json +222 -0
  8. freemyid/cli.py +725 -0
  9. freemyid/cli_json.py +198 -0
  10. freemyid/config.py +43 -0
  11. freemyid/consent.py +82 -0
  12. freemyid/crypto.py +197 -0
  13. freemyid/doctor.py +374 -0
  14. freemyid/engine/__init__.py +0 -0
  15. freemyid/engine/_dev/__init__.py +101 -0
  16. freemyid/engine/_dev/devrun.py +101 -0
  17. freemyid/engine/_dev/inbox/devauto.eml +7 -0
  18. freemyid/engine/_dev/registry.yaml +106 -0
  19. freemyid/engine/_dev/site/devauto.example/confirm.html +1 -0
  20. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/il.html +17 -0
  21. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/nc.html +4 -0
  22. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/oh.html +4 -0
  23. freemyid/engine/_dev/site/devauto.example/optout/done.html +1 -0
  24. freemyid/engine/_dev/site/devauto.example/optout.html +6 -0
  25. freemyid/engine/_dev/site/devblocked.example/name/jane-testperson/il.html +3 -0
  26. freemyid/engine/_dev/site/devblocked.example/optout.html +3 -0
  27. freemyid/engine/_dev/site/devfill.example/ajax.html +20 -0
  28. freemyid/engine/_dev/site/devfill.example/aria.html +16 -0
  29. freemyid/engine/_dev/site/devfill.example/consent.html +20 -0
  30. freemyid/engine/_dev/site/devfill.example/done.html +3 -0
  31. freemyid/engine/_dev/site/devfill.example/empty.html +1 -0
  32. freemyid/engine/_dev/site/devfill.example/fullname.html +12 -0
  33. freemyid/engine/_dev/site/devfill.example/labels.html +32 -0
  34. freemyid/engine/_dev/site/devfill.example/late.html +22 -0
  35. freemyid/engine/_dev/site/devfind.example/hostile.html +12 -0
  36. freemyid/engine/_dev/site/devfind.example/p/jane-testperson-shelbyville-il.html +6 -0
  37. freemyid/engine/_dev/site/devfind.example/p/jane-testperson-springfield-il.html +6 -0
  38. freemyid/engine/_dev/site/devfind.example/results.html +11 -0
  39. freemyid/engine/_dev/site/devfind.example/search.html +16 -0
  40. freemyid/engine/_dev/site/devweb.example/remove.html +1 -0
  41. freemyid/engine/browser.py +430 -0
  42. freemyid/engine/fill_for_me.py +630 -0
  43. freemyid/engine/filler.py +523 -0
  44. freemyid/engine/handoff.py +237 -0
  45. freemyid/engine/letters.py +125 -0
  46. freemyid/engine/listing_finder.py +538 -0
  47. freemyid/engine/mail.py +406 -0
  48. freemyid/engine/mail_tls.py +198 -0
  49. freemyid/engine/matcher.py +208 -0
  50. freemyid/engine/outbox.py +421 -0
  51. freemyid/engine/plugins.py +68 -0
  52. freemyid/engine/runner.py +791 -0
  53. freemyid/engine/tiers.py +107 -0
  54. freemyid/interfaces.py +164 -0
  55. freemyid/logging_setup.py +152 -0
  56. freemyid/models.py +700 -0
  57. freemyid/netpolicy.py +145 -0
  58. freemyid/notify.py +58 -0
  59. freemyid/paths.py +83 -0
  60. freemyid/registry.py +275 -0
  61. freemyid/scheduler/__init__.py +203 -0
  62. freemyid/scheduler/render.py +91 -0
  63. freemyid/secret_backends.py +562 -0
  64. freemyid/secrets.py +25 -0
  65. freemyid/setup_config.py +311 -0
  66. freemyid/state.py +151 -0
  67. freemyid/storage/__init__.py +3 -0
  68. freemyid/storage/db.py +394 -0
  69. freemyid/submitters/__init__.py +0 -0
  70. freemyid/submitters/assisted.py +171 -0
  71. freemyid/submitters/base.py +119 -0
  72. freemyid/submitters/email.py +40 -0
  73. freemyid/submitters/web_form.py +276 -0
  74. freemyid/templates/_identity.txt.j2 +19 -0
  75. freemyid/templates/_signature.txt.j2 +7 -0
  76. freemyid/templates/ccpa.txt.j2 +16 -0
  77. freemyid/templates/follow_up.txt.j2 +15 -0
  78. freemyid/templates/generic_removal.txt.j2 +12 -0
  79. freemyid/templates/legal_demand.txt.j2 +20 -0
  80. freemyid/templates/phone_script.txt.j2 +25 -0
  81. freemyid/templates/postal_letter.txt.j2 +20 -0
  82. freemyid/templates/state_privacy.txt.j2 +17 -0
  83. freemyid/wizard/__init__.py +0 -0
  84. freemyid/wizard/dashboard.py +401 -0
  85. freemyid/wizard/desktop.py +188 -0
  86. freemyid/wizard/draft.py +627 -0
  87. freemyid/wizard/firstrun.py +42 -0
  88. freemyid/wizard/flows.py +283 -0
  89. freemyid/wizard/report.py +86 -0
  90. freemyid/wizard/server.py +834 -0
  91. freemyid/wizard/static/__init__.py +0 -0
  92. freemyid/wizard/static/app.js +98 -0
  93. freemyid/wizard/static/freemyid.png +0 -0
  94. freemyid/wizard/static/style.css +85 -0
  95. freemyid/wizard/ui.py +343 -0
  96. freemyid-0.2.0.dev1.dist-info/METADATA +396 -0
  97. freemyid-0.2.0.dev1.dist-info/RECORD +100 -0
  98. freemyid-0.2.0.dev1.dist-info/WHEEL +4 -0
  99. freemyid-0.2.0.dev1.dist-info/entry_points.txt +2 -0
  100. freemyid-0.2.0.dev1.dist-info/licenses/LICENSE +661 -0
@@ -0,0 +1,119 @@
1
+ """Shared helpers for submitters: data minimization and evidence paths.
2
+
3
+ DATA MINIMIZATION (shared rule 6): a broker only ever gets
4
+ * fields its listings already show (Listing.shown_fields), intersected with what the
5
+ broker needs (Broker.required_fields, or name/city/state/url/email if unspecified);
6
+ * for a "blind" broker (no search, so no listing): the name, plus city/state only if the
7
+ broker requires them;
8
+ * the removal mailbox (never a personal email);
9
+ * a phone number only if shown AND required AND permissions.allow_phone_disclosure.
10
+ DOB, relatives and personal emails are never included.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ from types import SimpleNamespace
15
+
16
+ from ..models import Broker, Evidence, EvidenceKind, Listing, MatchLevel, Profile, ProfileField
17
+
18
+ F = ProfileField
19
+ _DEFAULT_NEEDS = {F.FULL_NAME, F.CITY, F.STATE, F.LISTING_URL, F.EMAIL}
20
+ _BLIND_OK = {F.FULL_NAME, F.FIRST_NAME, F.LAST_NAME, F.MIDDLE_NAME, F.CITY, F.STATE,
21
+ F.LISTING_URL, F.EMAIL}
22
+ _NEVER = {F.DOB, F.ID_DOCUMENT}
23
+
24
+
25
+ def confirmed(listings: list[Listing]) -> list[Listing]:
26
+ return [li for li in listings if li.match_level == MatchLevel.CONFIRMED]
27
+
28
+
29
+ def allowed_fields(profile: Profile, broker: Broker, listings: list[Listing]) -> set[F]:
30
+ """Profile fields this broker may receive for these listings."""
31
+ needs = set(broker.required_fields) or set(_DEFAULT_NEEDS)
32
+ if F.FULL_NAME in needs:
33
+ needs |= {F.FIRST_NAME, F.LAST_NAME}
34
+ if listings:
35
+ shown = {f for li in listings for f in li.shown_fields} | {F.LISTING_URL, F.EMAIL}
36
+ if F.FULL_NAME in shown:
37
+ shown |= {F.FIRST_NAME, F.LAST_NAME}
38
+ else:
39
+ shown = set(_BLIND_OK)
40
+ out = (needs & shown) - _NEVER
41
+ if F.PHONE in out and not profile.permissions.allow_phone_disclosure:
42
+ out.discard(F.PHONE)
43
+ out.add(F.FULL_NAME) # the name is how any broker finds the record
44
+ return out
45
+
46
+
47
+ def _current(profile: Profile):
48
+ return next((a for a in profile.addresses if a.current), profile.addresses[0])
49
+
50
+
51
+ def copy_fields(profile: Profile, broker: Broker, listings: list[Listing]) -> dict[str, str]:
52
+ """Label -> value for the Assisted card's Copy buttons. Minimized (see module doc)."""
53
+ allowed = allowed_fields(profile, broker, listings)
54
+ cur = _current(profile)
55
+ out: dict[str, str] = {}
56
+ if F.FULL_NAME in allowed:
57
+ out["Full name"] = f"{profile.name.first} {profile.name.last}"
58
+ if F.FIRST_NAME in allowed and F.FIRST_NAME in broker.required_fields:
59
+ out["First name"] = profile.name.first
60
+ if F.LAST_NAME in allowed and F.LAST_NAME in broker.required_fields:
61
+ out["Last name"] = profile.name.last
62
+ if F.CITY in allowed:
63
+ out["City"] = cur.city
64
+ if F.STATE in allowed:
65
+ out["State"] = cur.state
66
+ if F.STREET_ADDRESS in allowed and cur.street:
67
+ out["Street address"] = cur.street
68
+ if F.ZIP in allowed and cur.zip:
69
+ out["ZIP code"] = cur.zip
70
+ if F.AGE in allowed and (age := profile.estimated_age()) is not None:
71
+ out["Age"] = str(age)
72
+ if F.PHONE in allowed and profile.phones:
73
+ out["Phone"] = profile.phones[0]
74
+ for i, li in enumerate(confirmed(listings) or listings):
75
+ out["Listing link" if i == 0 else f"Listing link {i + 1}"] = li.url
76
+ if F.EMAIL in allowed and profile.mailbox:
77
+ out["Email for removals"] = profile.mailbox.address
78
+ return out
79
+
80
+
81
+ def form_vars(profile: Profile, broker: Broker, listing: Listing | None) -> dict:
82
+ """Variables for FormStep.value templates. Only allowed fields exist; anything else is
83
+ undefined, so a form step asking for it fails (and the run drops to Assisted)."""
84
+ lis = [listing] if listing else []
85
+ allowed = allowed_fields(profile, broker, lis)
86
+ cur = _current(profile)
87
+ p: dict[str, object] = {}
88
+ if F.FULL_NAME in allowed or F.FIRST_NAME in allowed:
89
+ p.update(first=profile.name.first, last=profile.name.last,
90
+ full_name=f"{profile.name.first} {profile.name.last}")
91
+ if F.MIDDLE_NAME in allowed and profile.name.middle:
92
+ p["middle"] = profile.name.middle
93
+ if F.CITY in allowed:
94
+ p["city"] = cur.city
95
+ if F.STATE in allowed:
96
+ p["state"] = cur.state
97
+ if F.STREET_ADDRESS in allowed and cur.street:
98
+ p["street"] = cur.street
99
+ if F.ZIP in allowed and cur.zip:
100
+ p["zip"] = cur.zip
101
+ if F.AGE in allowed:
102
+ p["age"] = profile.estimated_age()
103
+ if F.PHONE in allowed and profile.phones:
104
+ p["phone"] = profile.phones[0]
105
+ return {
106
+ "profile": SimpleNamespace(**p),
107
+ "listing": SimpleNamespace(url=listing.url) if listing else None,
108
+ "mailbox": SimpleNamespace(address=profile.mailbox.address) if profile.mailbox else None,
109
+ "broker": SimpleNamespace(id=broker.id, name=broker.name),
110
+ }
111
+
112
+
113
+ def evidence(profile_id: str, broker_id: str, kind: EvidenceKind, ext: str, *,
114
+ listing_id: str | None = None, request_id: str | None = None,
115
+ caption: str | None = None) -> Evidence:
116
+ """Evidence record with relpath <profile_id>/<broker_id>/<id>.<ext> (sha set on save)."""
117
+ ev = Evidence(profile_id=profile_id, broker_id=broker_id, kind=kind, relpath="x",
118
+ sha256="", listing_id=listing_id, request_id=request_id, caption=caption)
119
+ return ev.model_copy(update={"relpath": f"{profile_id}/{broker_id}/{ev.id}.{ext}"})
@@ -0,0 +1,40 @@
1
+ """Email tier: privacy-request letter from the removal mailbox (via engine.outbox).
2
+
3
+ * dry run -> DRAFT request (dry_run=True), never sent.
4
+ * APPROVE_EACH -> QUEUED_FOR_APPROVAL; state unchanged until the user approves.
5
+ * AUTO -> sent now; the runner marks the broker SUBMITTED.
6
+ * no mailbox -> falls back to a SEND_EMAIL_MANUALLY to-do (D2).
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from ..interfaces import RunContext, Submitter
11
+ from ..models import Listing, ListingState, RequestKind, RequestStatus, SubmitOutcome
12
+ from . import assisted, base
13
+
14
+
15
+ class EmailSubmitter(Submitter):
16
+ method = "email"
17
+
18
+ async def search(self, ctx: RunContext) -> list[Listing]:
19
+ return [] # email brokers are not searched; the letter names the person
20
+
21
+ async def submit(self, ctx: RunContext, listings: list[Listing]) -> SubmitOutcome:
22
+ from ..engine.outbox import compose, queue_email
23
+ kind: RequestKind = ctx.scratch.get("kind", RequestKind.OPT_OUT)
24
+ original = ctx.scratch.get("original")
25
+ use = base.confirmed(listings) or listings
26
+ if ctx.profile.mailbox is None:
27
+ subject, body = compose(ctx.profile, ctx.broker, use, ctx.state_law, kind,
28
+ original=original)
29
+ todo = assisted.manual_email_todo(ctx.profile, ctx.broker, subject, body)
30
+ return SubmitOutcome(new_state=ListingState.NEEDS_HUMAN, todos=[todo],
31
+ message="manual_email")
32
+ req = queue_email(ctx.store, ctx.profile, ctx.broker, use, kind=kind,
33
+ state_law=ctx.state_law, dry_run=ctx.dry_run, mailer=ctx.mailer,
34
+ original=original, now=ctx.now)
35
+ if req.status == RequestStatus.SENT:
36
+ new = (ListingState.AWAITING_VERIFICATION if ctx.broker.needs_email_verification
37
+ else ListingState.SUBMITTED)
38
+ return SubmitOutcome(new_state=new, request=req, message="email_sent")
39
+ return SubmitOutcome(new_state=ListingState.FOUND, request=req,
40
+ message="email_draft" if ctx.dry_run else "email_queued")
@@ -0,0 +1,276 @@
1
+ """Automatic tier: data-driven search (`Broker.search`) and opt-out (`Broker.form`).
2
+
3
+ No per-broker Python. Every request goes through BrowserManager (NetPolicy allowlist,
4
+ honest UA, polite delays). A challenge page, an unexpected page or ANY error drops the
5
+ broker to the Assisted tier (an ASSISTED_OPT_OUT to-do) instead of failing the run. In an
6
+ interactive run a challenge is first handed to the person in a visible window
7
+ (HumanHandoff); Free My ID never solves it.
8
+
9
+ Page text is data: it is only matched against selectors / expected text, never obeyed.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import logging
14
+ import re
15
+ from typing import TYPE_CHECKING, Any
16
+ from urllib.parse import quote, urljoin, urlsplit
17
+
18
+ from jinja2 import StrictUndefined
19
+ from jinja2.sandbox import SandboxedEnvironment
20
+
21
+ from ..engine import matcher
22
+ from ..engine.browser import BlockedByPolicy, ChallengeEncountered
23
+ from ..interfaces import RunContext, Submitter
24
+ from ..models import (
25
+ Broker,
26
+ EvidenceKind,
27
+ FormStep,
28
+ Listing,
29
+ ListingState,
30
+ Method,
31
+ Profile,
32
+ Request,
33
+ RequestKind,
34
+ RequestStatus,
35
+ SubmitOutcome,
36
+ Todo,
37
+ TodoKind,
38
+ utcnow,
39
+ )
40
+ from ..netpolicy import _host_ok
41
+ from . import assisted, base
42
+
43
+ if TYPE_CHECKING:
44
+ from playwright.async_api import Page
45
+
46
+ from ..engine.browser import BrowserManager
47
+
48
+ log = logging.getLogger(__name__)
49
+
50
+ _env = SandboxedEnvironment(undefined=StrictUndefined, autoescape=False)
51
+ MAX_RESULTS = 25
52
+
53
+
54
+ class UnexpectedPage(Exception):
55
+ """The page did not look like the registry said it would (PII-free label)."""
56
+
57
+
58
+ class HumanStep(Exception):
59
+ """A `human` form step: a person has to do this part."""
60
+
61
+
62
+ # --------------------------------------------------------------------------- search
63
+
64
+
65
+ def _slug(s: str) -> str:
66
+ return quote(re.sub(r"\s+", "-", s.strip().lower()), safe="-")
67
+
68
+
69
+ def search_url(broker: Broker, profile: Profile) -> str | None:
70
+ """Fill Broker.search_url_pattern. Phone only with allow_phone_disclosure."""
71
+ if not broker.search_url_pattern:
72
+ return None
73
+ cur = next((a for a in profile.addresses if a.current), profile.addresses[0])
74
+ vals: dict[str, str | None] = {
75
+ "first": profile.name.first, "last": profile.name.last,
76
+ "middle": profile.name.middle, "city": cur.city, "state": cur.state,
77
+ "state_name": matcher.STATE_NAMES.get(cur.state, cur.state), "zip": cur.zip,
78
+ "phone": matcher.digits(profile.phones[0])
79
+ if profile.phones and profile.permissions.allow_phone_disclosure else None,
80
+ }
81
+ import string
82
+ fields = {f for _, f, _, _ in string.Formatter().parse(broker.search_url_pattern) if f}
83
+ if any(not vals.get(f) for f in fields):
84
+ return None
85
+ url = broker.search_url_pattern.format(**{f: _slug(str(vals[f])) for f in fields})
86
+ return url if _host_ok(urlsplit(url).hostname or "", set(broker.domains)) else None
87
+
88
+
89
+ async def _text(el: Any, sel: str | None) -> str | None:
90
+ if not sel:
91
+ return None
92
+ sub = await el.query_selector(sel)
93
+ if sub is None:
94
+ return None
95
+ t = (await sub.inner_text()).strip()
96
+ return t[:500] or None
97
+
98
+
99
+ def _split(t: str | None) -> list[str]:
100
+ if not t:
101
+ return []
102
+ return [p.strip() for p in re.split(r"[\n;|]+", t) if p.strip()][:20]
103
+
104
+
105
+ async def parse_results(page: Page, broker: Broker) -> list[matcher.Candidate]:
106
+ """Candidates from a results page using Broker.search selectors. Links off the broker's
107
+ domains are dropped."""
108
+ spec = broker.search
109
+ if spec is None or not spec.result_selector:
110
+ return []
111
+ out: list[matcher.Candidate] = []
112
+ for el in (await page.query_selector_all(spec.result_selector))[:MAX_RESULTS]:
113
+ href = None
114
+ if spec.link_selector:
115
+ a = await el.query_selector(spec.link_selector)
116
+ href = await a.get_attribute("href") if a else None
117
+ url = urljoin(page.url, href) if href else page.url
118
+ if not _host_ok(urlsplit(url).hostname or "", set(broker.domains)):
119
+ continue
120
+ name = await _text(el, spec.name_selector)
121
+ locs = _split(await _text(el, spec.location_selector))
122
+ out.append(matcher.Candidate(
123
+ url=url, name=name, age=matcher.parse_age(await _text(el, spec.age_selector)),
124
+ locations=locs, relatives=_split(await _text(el, spec.relatives_selector))))
125
+ return out
126
+
127
+
128
+ async def search_listings(ctx: RunContext) -> list[Listing]:
129
+ """Search one broker with the browser; stores a page screenshot in ctx.scratch."""
130
+ bm: BrowserManager = ctx.scratch["browser"]
131
+ url = search_url(ctx.broker, ctx.profile)
132
+ if url is None:
133
+ raise UnexpectedPage("search_url_unavailable")
134
+ bctx = await bm.new_context(ctx.broker.domains)
135
+ try:
136
+ page = await bctx.new_page()
137
+ resp = await bm.goto(page, url)
138
+ if resp is not None and resp.status >= 400:
139
+ # an error page is not "no results": never record NOT_FOUND from it
140
+ raise UnexpectedPage(f"search_http_{resp.status}")
141
+ spec = ctx.broker.search
142
+ html = await page.content()
143
+ if spec and spec.no_results_text and spec.no_results_text.lower() in html.lower():
144
+ return []
145
+ cands = await parse_results(page, ctx.broker)
146
+ if not cands and spec and spec.no_results_text:
147
+ # neither results nor the site's "no results" text: the page changed
148
+ raise UnexpectedPage("search_page_unrecognized")
149
+ listings = matcher.score_all(ctx.profile, ctx.broker.id, cands,
150
+ (ctx.now or utcnow()).date())
151
+ if listings:
152
+ ctx.scratch["search_screenshot"] = await page.screenshot(full_page=True)
153
+ return listings
154
+ finally:
155
+ await bctx.close()
156
+
157
+
158
+ # --------------------------------------------------------------------------- form
159
+
160
+
161
+ def render_value(tpl: str | None, vars: dict) -> str:
162
+ return _env.from_string(tpl or "").render(**vars)
163
+
164
+
165
+ async def run_steps(bm: BrowserManager, page: Page, steps: list[FormStep], vars: dict,
166
+ shots: list[bytes]) -> None:
167
+ """Execute declarative steps. Raises ChallengeEncountered / UnexpectedPage / HumanStep."""
168
+ for st in steps:
169
+ a = st.action
170
+ if a == "human":
171
+ raise HumanStep(st.note or "human step")
172
+ if a == "goto":
173
+ await bm.goto(page, render_value(st.url, vars), timeout_ms=st.timeout_ms)
174
+ continue
175
+ sel = st.selector or ""
176
+ if a in ("fill", "select", "check", "click", "wait_for", "submit") and not sel:
177
+ raise UnexpectedPage(f"step_{a}_without_selector")
178
+ if a == "fill":
179
+ await page.fill(sel, render_value(st.value, vars), timeout=st.timeout_ms)
180
+ elif a == "select":
181
+ await page.select_option(sel, render_value(st.value, vars),
182
+ timeout=st.timeout_ms)
183
+ elif a == "check":
184
+ await page.check(sel, timeout=st.timeout_ms)
185
+ elif a == "wait_for":
186
+ await page.wait_for_selector(sel, timeout=st.timeout_ms)
187
+ elif a in ("click", "submit"):
188
+ await page.click(sel, timeout=st.timeout_ms)
189
+ await page.wait_for_load_state("domcontentloaded", timeout=st.timeout_ms)
190
+ bm.check_escaped(page) # SEC-16: a submit may redirect off-list
191
+ await bm.raise_if_challenge(page)
192
+ elif a == "screenshot":
193
+ shots.append(await page.screenshot(full_page=True))
194
+ elif a == "expect_text":
195
+ want = render_value(st.value, vars)
196
+ if want.lower() not in (await page.inner_text("body")).lower():
197
+ raise UnexpectedPage("expected_text_missing")
198
+
199
+
200
+ class WebFormSubmitter(Submitter):
201
+ method = "web_form"
202
+
203
+ async def search(self, ctx: RunContext) -> list[Listing]:
204
+ return await search_listings(ctx)
205
+
206
+ async def submit(self, ctx: RunContext, listings: list[Listing]) -> SubmitOutcome:
207
+ if ctx.dry_run:
208
+ raise RuntimeError("submit() must never be called in a dry run")
209
+ now = ctx.now or utcnow()
210
+ use = base.confirmed(listings)
211
+ uses_listing = any("listing" in (s.value or "") + (s.url or "") for s in ctx.broker.form)
212
+ if uses_listing and not use:
213
+ return self._fallback(ctx, listings, "no_confirmed_listing")
214
+ targets: list[Listing | None] = list(use) if uses_listing else [use[0] if use else None]
215
+ req = Request(profile_id=ctx.profile.id, broker_id=ctx.broker.id,
216
+ kind=ctx.scratch.get("kind", RequestKind.OPT_OUT), method=Method.WEB_FORM,
217
+ listing_ids=[li.id for li in use], created_at=now)
218
+ evs = []
219
+ try:
220
+ for li in targets:
221
+ shots = await self._fill_one(ctx, li)
222
+ for i, png in enumerate(shots):
223
+ ev = base.evidence(ctx.profile.id, ctx.broker.id, EvidenceKind.SCREENSHOT,
224
+ "png", listing_id=li.id if li else None,
225
+ request_id=req.id,
226
+ caption="confirmation page" if i == len(shots) - 1
227
+ else "form step")
228
+ evs.append(ctx.store.save_evidence(ev, png))
229
+ except ChallengeEncountered as e:
230
+ return self._fallback(ctx, listings, "challenge", detail=e.kind)
231
+ except HumanStep:
232
+ return self._fallback(ctx, listings, "human_step")
233
+ except (UnexpectedPage, BlockedByPolicy) as e:
234
+ return self._fallback(ctx, listings, "error", detail=type(e).__name__)
235
+ except Exception as e: # noqa: BLE001 - any error drops to Assisted (D1)
236
+ return self._fallback(ctx, listings, "error", detail=type(e).__name__)
237
+ req = req.model_copy(update={"status": RequestStatus.SENT, "sent_at": now})
238
+ ctx.store.save_request(req)
239
+ new = (ListingState.AWAITING_VERIFICATION if ctx.broker.needs_email_verification
240
+ else ListingState.SUBMITTED)
241
+ return SubmitOutcome(new_state=new, request=req, evidence=evs, message="form_submitted")
242
+
243
+ async def _fill_one(self, ctx: RunContext, listing: Listing | None) -> list[bytes]:
244
+ bm: BrowserManager = ctx.scratch["browser"]
245
+ vars = base.form_vars(ctx.profile, ctx.broker, listing)
246
+ bctx = await bm.new_context(ctx.broker.domains)
247
+ shots: list[bytes] = []
248
+ try:
249
+ page = await bctx.new_page()
250
+ if ctx.broker.form[0].action != "goto" and ctx.broker.opt_out_url:
251
+ await bm.goto(page, str(ctx.broker.opt_out_url))
252
+ try:
253
+ await run_steps(bm, page, ctx.broker.form, vars, shots)
254
+ except ChallengeEncountered as e:
255
+ if not ctx.interactive:
256
+ raise
257
+ todo = Todo(profile_id=ctx.profile.id, broker_id=ctx.broker.id,
258
+ kind=TodoKind.CAPTCHA,
259
+ title=f"{ctx.broker.name} wants a human check",
260
+ body="Finish the check in the window that opened, then come back.",
261
+ links=[page.url])
262
+ if not await ctx.handoff.request(todo, page):
263
+ raise
264
+ await bm.raise_if_challenge(page)
265
+ del e
266
+ shots.append(await page.screenshot(full_page=True))
267
+ return shots
268
+ finally:
269
+ await bctx.close()
270
+
271
+ def _fallback(self, ctx: RunContext, listings: list[Listing], reason: str,
272
+ detail: str | None = None) -> SubmitOutcome:
273
+ todo = assisted.todo_for(ctx.profile, ctx.broker, listings, ctx.state_law,
274
+ reason="challenge" if reason == "challenge" else "error")
275
+ msg = f"fallback_assisted:{reason}" + (f":{detail}" if detail else "")
276
+ return SubmitOutcome(new_state=ListingState.NEEDS_HUMAN, todos=[todo], message=msg)
@@ -0,0 +1,19 @@
1
+ {# Shared identifying block. Only contains fields the listing already shows. #}
2
+ Name: {{ name }}
3
+ {% if city or state %}Location shown on your site: {{ [city, state] | select | join(", ") }}
4
+ {% endif %}
5
+ {% if age %}Approximate age shown: {{ age }}
6
+ {% endif %}
7
+ {% if street_address %}Street address shown: {{ street_address }}{% if zip %} {{ zip }}{% endif %}
8
+
9
+ {% endif %}
10
+ {% if phone %}Phone number shown: {{ phone }}
11
+ {% endif %}
12
+ {% if listing_urls %}
13
+ Listing URL(s):
14
+ {% for u in listing_urls %}
15
+ - {{ u }}
16
+ {% endfor %}
17
+ {% endif %}
18
+ {% if mailbox %}Contact email for this request: {{ mailbox }}
19
+ {% endif %}
@@ -0,0 +1,7 @@
1
+ {% if is_agent and agent_name %}
2
+ Submitted by {{ agent_name }} as authorized agent / guardian on behalf of {{ name }}.
3
+ Proof of authorization is available on request.
4
+ {% else %}
5
+ Sincerely,
6
+ {{ name }}
7
+ {% endif %}
@@ -0,0 +1,16 @@
1
+ Subject: CCPA/CPRA request to delete and to opt out of sale/sharing
2
+ To {{ broker_name or "the privacy team" }},
3
+
4
+ I am a California resident. Under the {{ statute_name or "California Consumer Privacy Act, as amended by the CPRA" }}
5
+ ({{ citation or "Cal. Civ. Code §§ 1798.105, 1798.120" }}), I request that you:
6
+
7
+ 1. Delete all personal information you hold about me (right to delete); and
8
+ 2. Stop selling and sharing my personal information (right to opt out of sale/sharing).
9
+
10
+ {% include "_identity.txt.j2" %}
11
+
12
+ Please also direct any service providers and contractors to delete this information.
13
+ You must respond within {{ response_days or 45 }} days of receiving this request.
14
+ Please do not ask for more information than is needed to locate the records above.
15
+
16
+ {% include "_signature.txt.j2" %}
@@ -0,0 +1,15 @@
1
+ Subject: Follow-up: deletion request{% if original_request_date %} of {{ original_request_date }}{% endif %}{% if reference %} (ref {{ reference }}){% endif %}
2
+
3
+ To {{ broker_name or "the privacy team" }},
4
+
5
+ On {{ original_request_date or "an earlier date" }} I asked you to remove my personal information.
6
+ {% if response_deadline %}The response period ended on {{ response_deadline }}. {% endif %}My information still appears
7
+ on your site, or I have not received confirmation that it was removed.
8
+
9
+ {% include "_identity.txt.j2" %}
10
+
11
+ {% if statute_name %}This request is made under the {{ statute_name }}{% if citation %} ({{ citation }}){% endif %}.
12
+ {% endif %}
13
+ Please complete the deletion and confirm in writing.
14
+
15
+ {% include "_signature.txt.j2" %}
@@ -0,0 +1,12 @@
1
+ Subject: Request to remove my personal information
2
+ To {{ broker_name or "the privacy team" }},
3
+
4
+ I am asking you to remove my personal information from your website and records, and to
5
+ stop selling or sharing it. The record(s) concerned:
6
+
7
+ {% include "_identity.txt.j2" %}
8
+
9
+ Please delete these records, suppress them from future publication, and confirm by email
10
+ when this is done.
11
+
12
+ {% include "_signature.txt.j2" %}
@@ -0,0 +1,20 @@
1
+ Subject: Formal demand: unanswered personal data deletion request{% if reference %} (ref {{ reference }}){% endif %}
2
+
3
+ To {{ broker_name or "the privacy team" }} (legal / privacy officer),
4
+
5
+ This is a formal demand. On {{ original_request_date or "an earlier date" }} I requested deletion of my personal
6
+ information{% if statute_name %} under the {{ statute_name }}{% if citation %} ({{ citation }}){% endif %}{% endif %}.
7
+ {% if response_deadline %}Your response was due by {{ response_deadline }}. {% endif %}I have also sent a follow-up.
8
+ The information remains published or unconfirmed.
9
+
10
+ {% include "_identity.txt.j2" %}
11
+
12
+ Unless the deletion is completed and confirmed in writing within 15 days, I intend to file
13
+ a complaint with the appropriate regulator{% if complaint_urls %}:
14
+ {% for u in complaint_urls %}
15
+ - {{ u }}
16
+ {% endfor %}
17
+ {% else %}, including my state Attorney General.
18
+ {% endif %}
19
+
20
+ {% include "_signature.txt.j2" %}
@@ -0,0 +1,25 @@
1
+ Phone call script: {{ broker_name }}
2
+ {% if broker_phone %}Number: {{ broker_phone }}
3
+ {% endif %}
4
+
5
+ 1. "Hello, I'm calling to have my personal information removed from your records and
6
+ to opt out of the sale or sharing of my information."
7
+ 2. If asked for identifying details, give ONLY what their listing already shows:
8
+ - Name: {{ name }}
9
+ {% if city or state %} - Location: {{ [city, state] | select | join(", ") }}
10
+ {% endif %}
11
+ {% if age %} - Approximate age: {{ age }}
12
+ {% endif %}
13
+ {% for u in listing_urls %} - Listing: {{ u }}
14
+ {% endfor %}
15
+ {% if mailbox %} - Email for confirmation: {{ mailbox }}
16
+ {% endif %}
17
+ 3. Do NOT give your date of birth, Social Security number, payment details or relatives.
18
+ {% if not phone %} Do not give a phone number (phone disclosure is turned off for this person).
19
+ {% endif %}
20
+ 4. Ask for a reference / ticket number and write it down: ________________
21
+ 5. Ask how long removal takes and note the date: ________________
22
+ 6. Never agree to pay. Removal requests are free.
23
+ {% if statute_name %}
24
+ If they refuse: "I'm making this request under the {{ statute_name }}{% if citation %}, {{ citation }}{% endif %}."
25
+ {% endif %}
@@ -0,0 +1,20 @@
1
+ {{ today }}
2
+
3
+ {% if broker_postal %}{{ broker_postal }}
4
+ {% else %}{{ broker_name }}
5
+ Attn: Privacy / Opt-Out
6
+ {% endif %}
7
+
8
+ Re: Request to delete personal information and opt out of sale
9
+
10
+ To {{ broker_name or "the privacy team" }},
11
+
12
+ I am asking you to remove my personal information from your website and records, and to
13
+ stop selling or sharing it. The record(s) concerned:
14
+
15
+ {% include "_identity.txt.j2" %}
16
+
17
+ Please delete these records, suppress them from future publication, and confirm in writing
18
+ when this is done.
19
+
20
+ {% include "_signature.txt.j2" %}
@@ -0,0 +1,17 @@
1
+ Subject: Request to delete personal data{% if statute_name %} under the {{ statute_name }}{% endif %}
2
+
3
+ To {{ broker_name or "the privacy team" }},
4
+
5
+ I am a resident of {{ state_name or state or "the United States" }}.{% if statute_name %} Under the
6
+ {{ statute_name }}{% if citation %} ({{ citation }}){% endif %}, I am exercising my right to have my personal data
7
+ deleted and to opt out of its sale and use for targeted advertising.{% else %} I request that you
8
+ delete my personal data and stop selling or sharing it.{% endif %}
9
+
10
+
11
+ {% include "_identity.txt.j2" %}
12
+
13
+ {% if response_days %}The law requires a response within {{ response_days }} days.
14
+ {% endif %}
15
+ Please confirm by email when the deletion is complete.
16
+
17
+ {% include "_signature.txt.j2" %}
File without changes