freemyid 0.2.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- freemyid/__init__.py +2 -0
- freemyid/_data/brokers/LICENSE-DATA.md +28 -0
- freemyid/_data/brokers/registry.sample.yaml +674 -0
- freemyid/_data/brokers/state_laws.yaml +448 -0
- freemyid/_data/schema/registry.schema.json +668 -0
- freemyid/_data/schema/setup-config.schema.json +538 -0
- freemyid/_data/schema/state_laws.schema.json +222 -0
- freemyid/cli.py +725 -0
- freemyid/cli_json.py +198 -0
- freemyid/config.py +43 -0
- freemyid/consent.py +82 -0
- freemyid/crypto.py +197 -0
- freemyid/doctor.py +374 -0
- freemyid/engine/__init__.py +0 -0
- freemyid/engine/_dev/__init__.py +101 -0
- freemyid/engine/_dev/devrun.py +101 -0
- freemyid/engine/_dev/inbox/devauto.eml +7 -0
- freemyid/engine/_dev/registry.yaml +106 -0
- freemyid/engine/_dev/site/devauto.example/confirm.html +1 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/il.html +17 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/nc.html +4 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/oh.html +4 -0
- freemyid/engine/_dev/site/devauto.example/optout/done.html +1 -0
- freemyid/engine/_dev/site/devauto.example/optout.html +6 -0
- freemyid/engine/_dev/site/devblocked.example/name/jane-testperson/il.html +3 -0
- freemyid/engine/_dev/site/devblocked.example/optout.html +3 -0
- freemyid/engine/_dev/site/devfill.example/ajax.html +20 -0
- freemyid/engine/_dev/site/devfill.example/aria.html +16 -0
- freemyid/engine/_dev/site/devfill.example/consent.html +20 -0
- freemyid/engine/_dev/site/devfill.example/done.html +3 -0
- freemyid/engine/_dev/site/devfill.example/empty.html +1 -0
- freemyid/engine/_dev/site/devfill.example/fullname.html +12 -0
- freemyid/engine/_dev/site/devfill.example/labels.html +32 -0
- freemyid/engine/_dev/site/devfill.example/late.html +22 -0
- freemyid/engine/_dev/site/devfind.example/hostile.html +12 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-shelbyville-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-springfield-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/results.html +11 -0
- freemyid/engine/_dev/site/devfind.example/search.html +16 -0
- freemyid/engine/_dev/site/devweb.example/remove.html +1 -0
- freemyid/engine/browser.py +430 -0
- freemyid/engine/fill_for_me.py +630 -0
- freemyid/engine/filler.py +523 -0
- freemyid/engine/handoff.py +237 -0
- freemyid/engine/letters.py +125 -0
- freemyid/engine/listing_finder.py +538 -0
- freemyid/engine/mail.py +406 -0
- freemyid/engine/mail_tls.py +198 -0
- freemyid/engine/matcher.py +208 -0
- freemyid/engine/outbox.py +421 -0
- freemyid/engine/plugins.py +68 -0
- freemyid/engine/runner.py +791 -0
- freemyid/engine/tiers.py +107 -0
- freemyid/interfaces.py +164 -0
- freemyid/logging_setup.py +152 -0
- freemyid/models.py +700 -0
- freemyid/netpolicy.py +145 -0
- freemyid/notify.py +58 -0
- freemyid/paths.py +83 -0
- freemyid/registry.py +275 -0
- freemyid/scheduler/__init__.py +203 -0
- freemyid/scheduler/render.py +91 -0
- freemyid/secret_backends.py +562 -0
- freemyid/secrets.py +25 -0
- freemyid/setup_config.py +311 -0
- freemyid/state.py +151 -0
- freemyid/storage/__init__.py +3 -0
- freemyid/storage/db.py +394 -0
- freemyid/submitters/__init__.py +0 -0
- freemyid/submitters/assisted.py +171 -0
- freemyid/submitters/base.py +119 -0
- freemyid/submitters/email.py +40 -0
- freemyid/submitters/web_form.py +276 -0
- freemyid/templates/_identity.txt.j2 +19 -0
- freemyid/templates/_signature.txt.j2 +7 -0
- freemyid/templates/ccpa.txt.j2 +16 -0
- freemyid/templates/follow_up.txt.j2 +15 -0
- freemyid/templates/generic_removal.txt.j2 +12 -0
- freemyid/templates/legal_demand.txt.j2 +20 -0
- freemyid/templates/phone_script.txt.j2 +25 -0
- freemyid/templates/postal_letter.txt.j2 +20 -0
- freemyid/templates/state_privacy.txt.j2 +17 -0
- freemyid/wizard/__init__.py +0 -0
- freemyid/wizard/dashboard.py +401 -0
- freemyid/wizard/desktop.py +188 -0
- freemyid/wizard/draft.py +627 -0
- freemyid/wizard/firstrun.py +42 -0
- freemyid/wizard/flows.py +283 -0
- freemyid/wizard/report.py +86 -0
- freemyid/wizard/server.py +834 -0
- freemyid/wizard/static/__init__.py +0 -0
- freemyid/wizard/static/app.js +98 -0
- freemyid/wizard/static/freemyid.png +0 -0
- freemyid/wizard/static/style.css +85 -0
- freemyid/wizard/ui.py +343 -0
- freemyid-0.2.0.dev1.dist-info/METADATA +396 -0
- freemyid-0.2.0.dev1.dist-info/RECORD +100 -0
- freemyid-0.2.0.dev1.dist-info/WHEEL +4 -0
- freemyid-0.2.0.dev1.dist-info/entry_points.txt +2 -0
- freemyid-0.2.0.dev1.dist-info/licenses/LICENSE +661 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Shared helpers for submitters: data minimization and evidence paths.
|
|
2
|
+
|
|
3
|
+
DATA MINIMIZATION (shared rule 6): a broker only ever gets
|
|
4
|
+
* fields its listings already show (Listing.shown_fields), intersected with what the
|
|
5
|
+
broker needs (Broker.required_fields, or name/city/state/url/email if unspecified);
|
|
6
|
+
* for a "blind" broker (no search, so no listing): the name, plus city/state only if the
|
|
7
|
+
broker requires them;
|
|
8
|
+
* the removal mailbox (never a personal email);
|
|
9
|
+
* a phone number only if shown AND required AND permissions.allow_phone_disclosure.
|
|
10
|
+
DOB, relatives and personal emails are never included.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from types import SimpleNamespace
|
|
15
|
+
|
|
16
|
+
from ..models import Broker, Evidence, EvidenceKind, Listing, MatchLevel, Profile, ProfileField
|
|
17
|
+
|
|
18
|
+
F = ProfileField
|
|
19
|
+
_DEFAULT_NEEDS = {F.FULL_NAME, F.CITY, F.STATE, F.LISTING_URL, F.EMAIL}
|
|
20
|
+
_BLIND_OK = {F.FULL_NAME, F.FIRST_NAME, F.LAST_NAME, F.MIDDLE_NAME, F.CITY, F.STATE,
|
|
21
|
+
F.LISTING_URL, F.EMAIL}
|
|
22
|
+
_NEVER = {F.DOB, F.ID_DOCUMENT}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def confirmed(listings: list[Listing]) -> list[Listing]:
|
|
26
|
+
return [li for li in listings if li.match_level == MatchLevel.CONFIRMED]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def allowed_fields(profile: Profile, broker: Broker, listings: list[Listing]) -> set[F]:
|
|
30
|
+
"""Profile fields this broker may receive for these listings."""
|
|
31
|
+
needs = set(broker.required_fields) or set(_DEFAULT_NEEDS)
|
|
32
|
+
if F.FULL_NAME in needs:
|
|
33
|
+
needs |= {F.FIRST_NAME, F.LAST_NAME}
|
|
34
|
+
if listings:
|
|
35
|
+
shown = {f for li in listings for f in li.shown_fields} | {F.LISTING_URL, F.EMAIL}
|
|
36
|
+
if F.FULL_NAME in shown:
|
|
37
|
+
shown |= {F.FIRST_NAME, F.LAST_NAME}
|
|
38
|
+
else:
|
|
39
|
+
shown = set(_BLIND_OK)
|
|
40
|
+
out = (needs & shown) - _NEVER
|
|
41
|
+
if F.PHONE in out and not profile.permissions.allow_phone_disclosure:
|
|
42
|
+
out.discard(F.PHONE)
|
|
43
|
+
out.add(F.FULL_NAME) # the name is how any broker finds the record
|
|
44
|
+
return out
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _current(profile: Profile):
|
|
48
|
+
return next((a for a in profile.addresses if a.current), profile.addresses[0])
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def copy_fields(profile: Profile, broker: Broker, listings: list[Listing]) -> dict[str, str]:
|
|
52
|
+
"""Label -> value for the Assisted card's Copy buttons. Minimized (see module doc)."""
|
|
53
|
+
allowed = allowed_fields(profile, broker, listings)
|
|
54
|
+
cur = _current(profile)
|
|
55
|
+
out: dict[str, str] = {}
|
|
56
|
+
if F.FULL_NAME in allowed:
|
|
57
|
+
out["Full name"] = f"{profile.name.first} {profile.name.last}"
|
|
58
|
+
if F.FIRST_NAME in allowed and F.FIRST_NAME in broker.required_fields:
|
|
59
|
+
out["First name"] = profile.name.first
|
|
60
|
+
if F.LAST_NAME in allowed and F.LAST_NAME in broker.required_fields:
|
|
61
|
+
out["Last name"] = profile.name.last
|
|
62
|
+
if F.CITY in allowed:
|
|
63
|
+
out["City"] = cur.city
|
|
64
|
+
if F.STATE in allowed:
|
|
65
|
+
out["State"] = cur.state
|
|
66
|
+
if F.STREET_ADDRESS in allowed and cur.street:
|
|
67
|
+
out["Street address"] = cur.street
|
|
68
|
+
if F.ZIP in allowed and cur.zip:
|
|
69
|
+
out["ZIP code"] = cur.zip
|
|
70
|
+
if F.AGE in allowed and (age := profile.estimated_age()) is not None:
|
|
71
|
+
out["Age"] = str(age)
|
|
72
|
+
if F.PHONE in allowed and profile.phones:
|
|
73
|
+
out["Phone"] = profile.phones[0]
|
|
74
|
+
for i, li in enumerate(confirmed(listings) or listings):
|
|
75
|
+
out["Listing link" if i == 0 else f"Listing link {i + 1}"] = li.url
|
|
76
|
+
if F.EMAIL in allowed and profile.mailbox:
|
|
77
|
+
out["Email for removals"] = profile.mailbox.address
|
|
78
|
+
return out
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def form_vars(profile: Profile, broker: Broker, listing: Listing | None) -> dict:
|
|
82
|
+
"""Variables for FormStep.value templates. Only allowed fields exist; anything else is
|
|
83
|
+
undefined, so a form step asking for it fails (and the run drops to Assisted)."""
|
|
84
|
+
lis = [listing] if listing else []
|
|
85
|
+
allowed = allowed_fields(profile, broker, lis)
|
|
86
|
+
cur = _current(profile)
|
|
87
|
+
p: dict[str, object] = {}
|
|
88
|
+
if F.FULL_NAME in allowed or F.FIRST_NAME in allowed:
|
|
89
|
+
p.update(first=profile.name.first, last=profile.name.last,
|
|
90
|
+
full_name=f"{profile.name.first} {profile.name.last}")
|
|
91
|
+
if F.MIDDLE_NAME in allowed and profile.name.middle:
|
|
92
|
+
p["middle"] = profile.name.middle
|
|
93
|
+
if F.CITY in allowed:
|
|
94
|
+
p["city"] = cur.city
|
|
95
|
+
if F.STATE in allowed:
|
|
96
|
+
p["state"] = cur.state
|
|
97
|
+
if F.STREET_ADDRESS in allowed and cur.street:
|
|
98
|
+
p["street"] = cur.street
|
|
99
|
+
if F.ZIP in allowed and cur.zip:
|
|
100
|
+
p["zip"] = cur.zip
|
|
101
|
+
if F.AGE in allowed:
|
|
102
|
+
p["age"] = profile.estimated_age()
|
|
103
|
+
if F.PHONE in allowed and profile.phones:
|
|
104
|
+
p["phone"] = profile.phones[0]
|
|
105
|
+
return {
|
|
106
|
+
"profile": SimpleNamespace(**p),
|
|
107
|
+
"listing": SimpleNamespace(url=listing.url) if listing else None,
|
|
108
|
+
"mailbox": SimpleNamespace(address=profile.mailbox.address) if profile.mailbox else None,
|
|
109
|
+
"broker": SimpleNamespace(id=broker.id, name=broker.name),
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def evidence(profile_id: str, broker_id: str, kind: EvidenceKind, ext: str, *,
|
|
114
|
+
listing_id: str | None = None, request_id: str | None = None,
|
|
115
|
+
caption: str | None = None) -> Evidence:
|
|
116
|
+
"""Evidence record with relpath <profile_id>/<broker_id>/<id>.<ext> (sha set on save)."""
|
|
117
|
+
ev = Evidence(profile_id=profile_id, broker_id=broker_id, kind=kind, relpath="x",
|
|
118
|
+
sha256="", listing_id=listing_id, request_id=request_id, caption=caption)
|
|
119
|
+
return ev.model_copy(update={"relpath": f"{profile_id}/{broker_id}/{ev.id}.{ext}"})
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Email tier: privacy-request letter from the removal mailbox (via engine.outbox).
|
|
2
|
+
|
|
3
|
+
* dry run -> DRAFT request (dry_run=True), never sent.
|
|
4
|
+
* APPROVE_EACH -> QUEUED_FOR_APPROVAL; state unchanged until the user approves.
|
|
5
|
+
* AUTO -> sent now; the runner marks the broker SUBMITTED.
|
|
6
|
+
* no mailbox -> falls back to a SEND_EMAIL_MANUALLY to-do (D2).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from ..interfaces import RunContext, Submitter
|
|
11
|
+
from ..models import Listing, ListingState, RequestKind, RequestStatus, SubmitOutcome
|
|
12
|
+
from . import assisted, base
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EmailSubmitter(Submitter):
|
|
16
|
+
method = "email"
|
|
17
|
+
|
|
18
|
+
async def search(self, ctx: RunContext) -> list[Listing]:
|
|
19
|
+
return [] # email brokers are not searched; the letter names the person
|
|
20
|
+
|
|
21
|
+
async def submit(self, ctx: RunContext, listings: list[Listing]) -> SubmitOutcome:
|
|
22
|
+
from ..engine.outbox import compose, queue_email
|
|
23
|
+
kind: RequestKind = ctx.scratch.get("kind", RequestKind.OPT_OUT)
|
|
24
|
+
original = ctx.scratch.get("original")
|
|
25
|
+
use = base.confirmed(listings) or listings
|
|
26
|
+
if ctx.profile.mailbox is None:
|
|
27
|
+
subject, body = compose(ctx.profile, ctx.broker, use, ctx.state_law, kind,
|
|
28
|
+
original=original)
|
|
29
|
+
todo = assisted.manual_email_todo(ctx.profile, ctx.broker, subject, body)
|
|
30
|
+
return SubmitOutcome(new_state=ListingState.NEEDS_HUMAN, todos=[todo],
|
|
31
|
+
message="manual_email")
|
|
32
|
+
req = queue_email(ctx.store, ctx.profile, ctx.broker, use, kind=kind,
|
|
33
|
+
state_law=ctx.state_law, dry_run=ctx.dry_run, mailer=ctx.mailer,
|
|
34
|
+
original=original, now=ctx.now)
|
|
35
|
+
if req.status == RequestStatus.SENT:
|
|
36
|
+
new = (ListingState.AWAITING_VERIFICATION if ctx.broker.needs_email_verification
|
|
37
|
+
else ListingState.SUBMITTED)
|
|
38
|
+
return SubmitOutcome(new_state=new, request=req, message="email_sent")
|
|
39
|
+
return SubmitOutcome(new_state=ListingState.FOUND, request=req,
|
|
40
|
+
message="email_draft" if ctx.dry_run else "email_queued")
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""Automatic tier: data-driven search (`Broker.search`) and opt-out (`Broker.form`).
|
|
2
|
+
|
|
3
|
+
No per-broker Python. Every request goes through BrowserManager (NetPolicy allowlist,
|
|
4
|
+
honest UA, polite delays). A challenge page, an unexpected page or ANY error drops the
|
|
5
|
+
broker to the Assisted tier (an ASSISTED_OPT_OUT to-do) instead of failing the run. In an
|
|
6
|
+
interactive run a challenge is first handed to the person in a visible window
|
|
7
|
+
(HumanHandoff); Free My ID never solves it.
|
|
8
|
+
|
|
9
|
+
Page text is data: it is only matched against selectors / expected text, never obeyed.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import re
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
from urllib.parse import quote, urljoin, urlsplit
|
|
17
|
+
|
|
18
|
+
from jinja2 import StrictUndefined
|
|
19
|
+
from jinja2.sandbox import SandboxedEnvironment
|
|
20
|
+
|
|
21
|
+
from ..engine import matcher
|
|
22
|
+
from ..engine.browser import BlockedByPolicy, ChallengeEncountered
|
|
23
|
+
from ..interfaces import RunContext, Submitter
|
|
24
|
+
from ..models import (
|
|
25
|
+
Broker,
|
|
26
|
+
EvidenceKind,
|
|
27
|
+
FormStep,
|
|
28
|
+
Listing,
|
|
29
|
+
ListingState,
|
|
30
|
+
Method,
|
|
31
|
+
Profile,
|
|
32
|
+
Request,
|
|
33
|
+
RequestKind,
|
|
34
|
+
RequestStatus,
|
|
35
|
+
SubmitOutcome,
|
|
36
|
+
Todo,
|
|
37
|
+
TodoKind,
|
|
38
|
+
utcnow,
|
|
39
|
+
)
|
|
40
|
+
from ..netpolicy import _host_ok
|
|
41
|
+
from . import assisted, base
|
|
42
|
+
|
|
43
|
+
if TYPE_CHECKING:
|
|
44
|
+
from playwright.async_api import Page
|
|
45
|
+
|
|
46
|
+
from ..engine.browser import BrowserManager
|
|
47
|
+
|
|
48
|
+
log = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
_env = SandboxedEnvironment(undefined=StrictUndefined, autoescape=False)
|
|
51
|
+
MAX_RESULTS = 25
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class UnexpectedPage(Exception):
|
|
55
|
+
"""The page did not look like the registry said it would (PII-free label)."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class HumanStep(Exception):
|
|
59
|
+
"""A `human` form step: a person has to do this part."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --------------------------------------------------------------------------- search
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _slug(s: str) -> str:
|
|
66
|
+
return quote(re.sub(r"\s+", "-", s.strip().lower()), safe="-")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def search_url(broker: Broker, profile: Profile) -> str | None:
|
|
70
|
+
"""Fill Broker.search_url_pattern. Phone only with allow_phone_disclosure."""
|
|
71
|
+
if not broker.search_url_pattern:
|
|
72
|
+
return None
|
|
73
|
+
cur = next((a for a in profile.addresses if a.current), profile.addresses[0])
|
|
74
|
+
vals: dict[str, str | None] = {
|
|
75
|
+
"first": profile.name.first, "last": profile.name.last,
|
|
76
|
+
"middle": profile.name.middle, "city": cur.city, "state": cur.state,
|
|
77
|
+
"state_name": matcher.STATE_NAMES.get(cur.state, cur.state), "zip": cur.zip,
|
|
78
|
+
"phone": matcher.digits(profile.phones[0])
|
|
79
|
+
if profile.phones and profile.permissions.allow_phone_disclosure else None,
|
|
80
|
+
}
|
|
81
|
+
import string
|
|
82
|
+
fields = {f for _, f, _, _ in string.Formatter().parse(broker.search_url_pattern) if f}
|
|
83
|
+
if any(not vals.get(f) for f in fields):
|
|
84
|
+
return None
|
|
85
|
+
url = broker.search_url_pattern.format(**{f: _slug(str(vals[f])) for f in fields})
|
|
86
|
+
return url if _host_ok(urlsplit(url).hostname or "", set(broker.domains)) else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
async def _text(el: Any, sel: str | None) -> str | None:
|
|
90
|
+
if not sel:
|
|
91
|
+
return None
|
|
92
|
+
sub = await el.query_selector(sel)
|
|
93
|
+
if sub is None:
|
|
94
|
+
return None
|
|
95
|
+
t = (await sub.inner_text()).strip()
|
|
96
|
+
return t[:500] or None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _split(t: str | None) -> list[str]:
|
|
100
|
+
if not t:
|
|
101
|
+
return []
|
|
102
|
+
return [p.strip() for p in re.split(r"[\n;|]+", t) if p.strip()][:20]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
async def parse_results(page: Page, broker: Broker) -> list[matcher.Candidate]:
|
|
106
|
+
"""Candidates from a results page using Broker.search selectors. Links off the broker's
|
|
107
|
+
domains are dropped."""
|
|
108
|
+
spec = broker.search
|
|
109
|
+
if spec is None or not spec.result_selector:
|
|
110
|
+
return []
|
|
111
|
+
out: list[matcher.Candidate] = []
|
|
112
|
+
for el in (await page.query_selector_all(spec.result_selector))[:MAX_RESULTS]:
|
|
113
|
+
href = None
|
|
114
|
+
if spec.link_selector:
|
|
115
|
+
a = await el.query_selector(spec.link_selector)
|
|
116
|
+
href = await a.get_attribute("href") if a else None
|
|
117
|
+
url = urljoin(page.url, href) if href else page.url
|
|
118
|
+
if not _host_ok(urlsplit(url).hostname or "", set(broker.domains)):
|
|
119
|
+
continue
|
|
120
|
+
name = await _text(el, spec.name_selector)
|
|
121
|
+
locs = _split(await _text(el, spec.location_selector))
|
|
122
|
+
out.append(matcher.Candidate(
|
|
123
|
+
url=url, name=name, age=matcher.parse_age(await _text(el, spec.age_selector)),
|
|
124
|
+
locations=locs, relatives=_split(await _text(el, spec.relatives_selector))))
|
|
125
|
+
return out
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
async def search_listings(ctx: RunContext) -> list[Listing]:
|
|
129
|
+
"""Search one broker with the browser; stores a page screenshot in ctx.scratch."""
|
|
130
|
+
bm: BrowserManager = ctx.scratch["browser"]
|
|
131
|
+
url = search_url(ctx.broker, ctx.profile)
|
|
132
|
+
if url is None:
|
|
133
|
+
raise UnexpectedPage("search_url_unavailable")
|
|
134
|
+
bctx = await bm.new_context(ctx.broker.domains)
|
|
135
|
+
try:
|
|
136
|
+
page = await bctx.new_page()
|
|
137
|
+
resp = await bm.goto(page, url)
|
|
138
|
+
if resp is not None and resp.status >= 400:
|
|
139
|
+
# an error page is not "no results": never record NOT_FOUND from it
|
|
140
|
+
raise UnexpectedPage(f"search_http_{resp.status}")
|
|
141
|
+
spec = ctx.broker.search
|
|
142
|
+
html = await page.content()
|
|
143
|
+
if spec and spec.no_results_text and spec.no_results_text.lower() in html.lower():
|
|
144
|
+
return []
|
|
145
|
+
cands = await parse_results(page, ctx.broker)
|
|
146
|
+
if not cands and spec and spec.no_results_text:
|
|
147
|
+
# neither results nor the site's "no results" text: the page changed
|
|
148
|
+
raise UnexpectedPage("search_page_unrecognized")
|
|
149
|
+
listings = matcher.score_all(ctx.profile, ctx.broker.id, cands,
|
|
150
|
+
(ctx.now or utcnow()).date())
|
|
151
|
+
if listings:
|
|
152
|
+
ctx.scratch["search_screenshot"] = await page.screenshot(full_page=True)
|
|
153
|
+
return listings
|
|
154
|
+
finally:
|
|
155
|
+
await bctx.close()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# --------------------------------------------------------------------------- form
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def render_value(tpl: str | None, vars: dict) -> str:
|
|
162
|
+
return _env.from_string(tpl or "").render(**vars)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
async def run_steps(bm: BrowserManager, page: Page, steps: list[FormStep], vars: dict,
|
|
166
|
+
shots: list[bytes]) -> None:
|
|
167
|
+
"""Execute declarative steps. Raises ChallengeEncountered / UnexpectedPage / HumanStep."""
|
|
168
|
+
for st in steps:
|
|
169
|
+
a = st.action
|
|
170
|
+
if a == "human":
|
|
171
|
+
raise HumanStep(st.note or "human step")
|
|
172
|
+
if a == "goto":
|
|
173
|
+
await bm.goto(page, render_value(st.url, vars), timeout_ms=st.timeout_ms)
|
|
174
|
+
continue
|
|
175
|
+
sel = st.selector or ""
|
|
176
|
+
if a in ("fill", "select", "check", "click", "wait_for", "submit") and not sel:
|
|
177
|
+
raise UnexpectedPage(f"step_{a}_without_selector")
|
|
178
|
+
if a == "fill":
|
|
179
|
+
await page.fill(sel, render_value(st.value, vars), timeout=st.timeout_ms)
|
|
180
|
+
elif a == "select":
|
|
181
|
+
await page.select_option(sel, render_value(st.value, vars),
|
|
182
|
+
timeout=st.timeout_ms)
|
|
183
|
+
elif a == "check":
|
|
184
|
+
await page.check(sel, timeout=st.timeout_ms)
|
|
185
|
+
elif a == "wait_for":
|
|
186
|
+
await page.wait_for_selector(sel, timeout=st.timeout_ms)
|
|
187
|
+
elif a in ("click", "submit"):
|
|
188
|
+
await page.click(sel, timeout=st.timeout_ms)
|
|
189
|
+
await page.wait_for_load_state("domcontentloaded", timeout=st.timeout_ms)
|
|
190
|
+
bm.check_escaped(page) # SEC-16: a submit may redirect off-list
|
|
191
|
+
await bm.raise_if_challenge(page)
|
|
192
|
+
elif a == "screenshot":
|
|
193
|
+
shots.append(await page.screenshot(full_page=True))
|
|
194
|
+
elif a == "expect_text":
|
|
195
|
+
want = render_value(st.value, vars)
|
|
196
|
+
if want.lower() not in (await page.inner_text("body")).lower():
|
|
197
|
+
raise UnexpectedPage("expected_text_missing")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class WebFormSubmitter(Submitter):
|
|
201
|
+
method = "web_form"
|
|
202
|
+
|
|
203
|
+
async def search(self, ctx: RunContext) -> list[Listing]:
|
|
204
|
+
return await search_listings(ctx)
|
|
205
|
+
|
|
206
|
+
async def submit(self, ctx: RunContext, listings: list[Listing]) -> SubmitOutcome:
|
|
207
|
+
if ctx.dry_run:
|
|
208
|
+
raise RuntimeError("submit() must never be called in a dry run")
|
|
209
|
+
now = ctx.now or utcnow()
|
|
210
|
+
use = base.confirmed(listings)
|
|
211
|
+
uses_listing = any("listing" in (s.value or "") + (s.url or "") for s in ctx.broker.form)
|
|
212
|
+
if uses_listing and not use:
|
|
213
|
+
return self._fallback(ctx, listings, "no_confirmed_listing")
|
|
214
|
+
targets: list[Listing | None] = list(use) if uses_listing else [use[0] if use else None]
|
|
215
|
+
req = Request(profile_id=ctx.profile.id, broker_id=ctx.broker.id,
|
|
216
|
+
kind=ctx.scratch.get("kind", RequestKind.OPT_OUT), method=Method.WEB_FORM,
|
|
217
|
+
listing_ids=[li.id for li in use], created_at=now)
|
|
218
|
+
evs = []
|
|
219
|
+
try:
|
|
220
|
+
for li in targets:
|
|
221
|
+
shots = await self._fill_one(ctx, li)
|
|
222
|
+
for i, png in enumerate(shots):
|
|
223
|
+
ev = base.evidence(ctx.profile.id, ctx.broker.id, EvidenceKind.SCREENSHOT,
|
|
224
|
+
"png", listing_id=li.id if li else None,
|
|
225
|
+
request_id=req.id,
|
|
226
|
+
caption="confirmation page" if i == len(shots) - 1
|
|
227
|
+
else "form step")
|
|
228
|
+
evs.append(ctx.store.save_evidence(ev, png))
|
|
229
|
+
except ChallengeEncountered as e:
|
|
230
|
+
return self._fallback(ctx, listings, "challenge", detail=e.kind)
|
|
231
|
+
except HumanStep:
|
|
232
|
+
return self._fallback(ctx, listings, "human_step")
|
|
233
|
+
except (UnexpectedPage, BlockedByPolicy) as e:
|
|
234
|
+
return self._fallback(ctx, listings, "error", detail=type(e).__name__)
|
|
235
|
+
except Exception as e: # noqa: BLE001 - any error drops to Assisted (D1)
|
|
236
|
+
return self._fallback(ctx, listings, "error", detail=type(e).__name__)
|
|
237
|
+
req = req.model_copy(update={"status": RequestStatus.SENT, "sent_at": now})
|
|
238
|
+
ctx.store.save_request(req)
|
|
239
|
+
new = (ListingState.AWAITING_VERIFICATION if ctx.broker.needs_email_verification
|
|
240
|
+
else ListingState.SUBMITTED)
|
|
241
|
+
return SubmitOutcome(new_state=new, request=req, evidence=evs, message="form_submitted")
|
|
242
|
+
|
|
243
|
+
async def _fill_one(self, ctx: RunContext, listing: Listing | None) -> list[bytes]:
|
|
244
|
+
bm: BrowserManager = ctx.scratch["browser"]
|
|
245
|
+
vars = base.form_vars(ctx.profile, ctx.broker, listing)
|
|
246
|
+
bctx = await bm.new_context(ctx.broker.domains)
|
|
247
|
+
shots: list[bytes] = []
|
|
248
|
+
try:
|
|
249
|
+
page = await bctx.new_page()
|
|
250
|
+
if ctx.broker.form[0].action != "goto" and ctx.broker.opt_out_url:
|
|
251
|
+
await bm.goto(page, str(ctx.broker.opt_out_url))
|
|
252
|
+
try:
|
|
253
|
+
await run_steps(bm, page, ctx.broker.form, vars, shots)
|
|
254
|
+
except ChallengeEncountered as e:
|
|
255
|
+
if not ctx.interactive:
|
|
256
|
+
raise
|
|
257
|
+
todo = Todo(profile_id=ctx.profile.id, broker_id=ctx.broker.id,
|
|
258
|
+
kind=TodoKind.CAPTCHA,
|
|
259
|
+
title=f"{ctx.broker.name} wants a human check",
|
|
260
|
+
body="Finish the check in the window that opened, then come back.",
|
|
261
|
+
links=[page.url])
|
|
262
|
+
if not await ctx.handoff.request(todo, page):
|
|
263
|
+
raise
|
|
264
|
+
await bm.raise_if_challenge(page)
|
|
265
|
+
del e
|
|
266
|
+
shots.append(await page.screenshot(full_page=True))
|
|
267
|
+
return shots
|
|
268
|
+
finally:
|
|
269
|
+
await bctx.close()
|
|
270
|
+
|
|
271
|
+
def _fallback(self, ctx: RunContext, listings: list[Listing], reason: str,
|
|
272
|
+
detail: str | None = None) -> SubmitOutcome:
|
|
273
|
+
todo = assisted.todo_for(ctx.profile, ctx.broker, listings, ctx.state_law,
|
|
274
|
+
reason="challenge" if reason == "challenge" else "error")
|
|
275
|
+
msg = f"fallback_assisted:{reason}" + (f":{detail}" if detail else "")
|
|
276
|
+
return SubmitOutcome(new_state=ListingState.NEEDS_HUMAN, todos=[todo], message=msg)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{# Shared identifying block. Only contains fields the listing already shows. #}
|
|
2
|
+
Name: {{ name }}
|
|
3
|
+
{% if city or state %}Location shown on your site: {{ [city, state] | select | join(", ") }}
|
|
4
|
+
{% endif %}
|
|
5
|
+
{% if age %}Approximate age shown: {{ age }}
|
|
6
|
+
{% endif %}
|
|
7
|
+
{% if street_address %}Street address shown: {{ street_address }}{% if zip %} {{ zip }}{% endif %}
|
|
8
|
+
|
|
9
|
+
{% endif %}
|
|
10
|
+
{% if phone %}Phone number shown: {{ phone }}
|
|
11
|
+
{% endif %}
|
|
12
|
+
{% if listing_urls %}
|
|
13
|
+
Listing URL(s):
|
|
14
|
+
{% for u in listing_urls %}
|
|
15
|
+
- {{ u }}
|
|
16
|
+
{% endfor %}
|
|
17
|
+
{% endif %}
|
|
18
|
+
{% if mailbox %}Contact email for this request: {{ mailbox }}
|
|
19
|
+
{% endif %}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
Subject: CCPA/CPRA request to delete and to opt out of sale/sharing
|
|
2
|
+
To {{ broker_name or "the privacy team" }},
|
|
3
|
+
|
|
4
|
+
I am a California resident. Under the {{ statute_name or "California Consumer Privacy Act, as amended by the CPRA" }}
|
|
5
|
+
({{ citation or "Cal. Civ. Code §§ 1798.105, 1798.120" }}), I request that you:
|
|
6
|
+
|
|
7
|
+
1. Delete all personal information you hold about me (right to delete); and
|
|
8
|
+
2. Stop selling and sharing my personal information (right to opt out of sale/sharing).
|
|
9
|
+
|
|
10
|
+
{% include "_identity.txt.j2" %}
|
|
11
|
+
|
|
12
|
+
Please also direct any service providers and contractors to delete this information.
|
|
13
|
+
You must respond within {{ response_days or 45 }} days of receiving this request.
|
|
14
|
+
Please do not ask for more information than is needed to locate the records above.
|
|
15
|
+
|
|
16
|
+
{% include "_signature.txt.j2" %}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Subject: Follow-up: deletion request{% if original_request_date %} of {{ original_request_date }}{% endif %}{% if reference %} (ref {{ reference }}){% endif %}
|
|
2
|
+
|
|
3
|
+
To {{ broker_name or "the privacy team" }},
|
|
4
|
+
|
|
5
|
+
On {{ original_request_date or "an earlier date" }} I asked you to remove my personal information.
|
|
6
|
+
{% if response_deadline %}The response period ended on {{ response_deadline }}. {% endif %}My information still appears
|
|
7
|
+
on your site, or I have not received confirmation that it was removed.
|
|
8
|
+
|
|
9
|
+
{% include "_identity.txt.j2" %}
|
|
10
|
+
|
|
11
|
+
{% if statute_name %}This request is made under the {{ statute_name }}{% if citation %} ({{ citation }}){% endif %}.
|
|
12
|
+
{% endif %}
|
|
13
|
+
Please complete the deletion and confirm in writing.
|
|
14
|
+
|
|
15
|
+
{% include "_signature.txt.j2" %}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Subject: Request to remove my personal information
|
|
2
|
+
To {{ broker_name or "the privacy team" }},
|
|
3
|
+
|
|
4
|
+
I am asking you to remove my personal information from your website and records, and to
|
|
5
|
+
stop selling or sharing it. The record(s) concerned:
|
|
6
|
+
|
|
7
|
+
{% include "_identity.txt.j2" %}
|
|
8
|
+
|
|
9
|
+
Please delete these records, suppress them from future publication, and confirm by email
|
|
10
|
+
when this is done.
|
|
11
|
+
|
|
12
|
+
{% include "_signature.txt.j2" %}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Subject: Formal demand: unanswered personal data deletion request{% if reference %} (ref {{ reference }}){% endif %}
|
|
2
|
+
|
|
3
|
+
To {{ broker_name or "the privacy team" }} (legal / privacy officer),
|
|
4
|
+
|
|
5
|
+
This is a formal demand. On {{ original_request_date or "an earlier date" }} I requested deletion of my personal
|
|
6
|
+
information{% if statute_name %} under the {{ statute_name }}{% if citation %} ({{ citation }}){% endif %}{% endif %}.
|
|
7
|
+
{% if response_deadline %}Your response was due by {{ response_deadline }}. {% endif %}I have also sent a follow-up.
|
|
8
|
+
The information remains published or unconfirmed.
|
|
9
|
+
|
|
10
|
+
{% include "_identity.txt.j2" %}
|
|
11
|
+
|
|
12
|
+
Unless the deletion is completed and confirmed in writing within 15 days, I intend to file
|
|
13
|
+
a complaint with the appropriate regulator{% if complaint_urls %}:
|
|
14
|
+
{% for u in complaint_urls %}
|
|
15
|
+
- {{ u }}
|
|
16
|
+
{% endfor %}
|
|
17
|
+
{% else %}, including my state Attorney General.
|
|
18
|
+
{% endif %}
|
|
19
|
+
|
|
20
|
+
{% include "_signature.txt.j2" %}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Phone call script: {{ broker_name }}
|
|
2
|
+
{% if broker_phone %}Number: {{ broker_phone }}
|
|
3
|
+
{% endif %}
|
|
4
|
+
|
|
5
|
+
1. "Hello, I'm calling to have my personal information removed from your records and
|
|
6
|
+
to opt out of the sale or sharing of my information."
|
|
7
|
+
2. If asked for identifying details, give ONLY what their listing already shows:
|
|
8
|
+
- Name: {{ name }}
|
|
9
|
+
{% if city or state %} - Location: {{ [city, state] | select | join(", ") }}
|
|
10
|
+
{% endif %}
|
|
11
|
+
{% if age %} - Approximate age: {{ age }}
|
|
12
|
+
{% endif %}
|
|
13
|
+
{% for u in listing_urls %} - Listing: {{ u }}
|
|
14
|
+
{% endfor %}
|
|
15
|
+
{% if mailbox %} - Email for confirmation: {{ mailbox }}
|
|
16
|
+
{% endif %}
|
|
17
|
+
3. Do NOT give your date of birth, Social Security number, payment details or relatives.
|
|
18
|
+
{% if not phone %} Do not give a phone number (phone disclosure is turned off for this person).
|
|
19
|
+
{% endif %}
|
|
20
|
+
4. Ask for a reference / ticket number and write it down: ________________
|
|
21
|
+
5. Ask how long removal takes and note the date: ________________
|
|
22
|
+
6. Never agree to pay. Removal requests are free.
|
|
23
|
+
{% if statute_name %}
|
|
24
|
+
If they refuse: "I'm making this request under the {{ statute_name }}{% if citation %}, {{ citation }}{% endif %}."
|
|
25
|
+
{% endif %}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{{ today }}
|
|
2
|
+
|
|
3
|
+
{% if broker_postal %}{{ broker_postal }}
|
|
4
|
+
{% else %}{{ broker_name }}
|
|
5
|
+
Attn: Privacy / Opt-Out
|
|
6
|
+
{% endif %}
|
|
7
|
+
|
|
8
|
+
Re: Request to delete personal information and opt out of sale
|
|
9
|
+
|
|
10
|
+
To {{ broker_name or "the privacy team" }},
|
|
11
|
+
|
|
12
|
+
I am asking you to remove my personal information from your website and records, and to
|
|
13
|
+
stop selling or sharing it. The record(s) concerned:
|
|
14
|
+
|
|
15
|
+
{% include "_identity.txt.j2" %}
|
|
16
|
+
|
|
17
|
+
Please delete these records, suppress them from future publication, and confirm in writing
|
|
18
|
+
when this is done.
|
|
19
|
+
|
|
20
|
+
{% include "_signature.txt.j2" %}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Subject: Request to delete personal data{% if statute_name %} under the {{ statute_name }}{% endif %}
|
|
2
|
+
|
|
3
|
+
To {{ broker_name or "the privacy team" }},
|
|
4
|
+
|
|
5
|
+
I am a resident of {{ state_name or state or "the United States" }}.{% if statute_name %} Under the
|
|
6
|
+
{{ statute_name }}{% if citation %} ({{ citation }}){% endif %}, I am exercising my right to have my personal data
|
|
7
|
+
deleted and to opt out of its sale and use for targeted advertising.{% else %} I request that you
|
|
8
|
+
delete my personal data and stop selling or sharing it.{% endif %}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
{% include "_identity.txt.j2" %}
|
|
12
|
+
|
|
13
|
+
{% if response_days %}The law requires a response within {{ response_days }} days.
|
|
14
|
+
{% endif %}
|
|
15
|
+
Please confirm by email when the deletion is complete.
|
|
16
|
+
|
|
17
|
+
{% include "_signature.txt.j2" %}
|
|
File without changes
|