amazon-subscriptions 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amazon_subscriptions/__init__.py +0 -0
- amazon_subscriptions/cli.py +180 -0
- amazon_subscriptions/client.py +485 -0
- amazon_subscriptions/dates.py +63 -0
- amazon_subscriptions/exceptions.py +21 -0
- amazon_subscriptions/locales/__init__.py +68 -0
- amazon_subscriptions/locales/base.py +189 -0
- amazon_subscriptions/locales/en.py +65 -0
- amazon_subscriptions/models.py +163 -0
- amazon_subscriptions/parse/__init__.py +28 -0
- amazon_subscriptions/parse/acp.py +41 -0
- amazon_subscriptions/parse/deliveries.py +199 -0
- amazon_subscriptions/parse/detail.py +27 -0
- amazon_subscriptions/parse/matching.py +100 -0
- amazon_subscriptions/parse/product.py +202 -0
- amazon_subscriptions/parse/selectors.py +64 -0
- amazon_subscriptions/parse/subscriptions.py +85 -0
- amazon_subscriptions/parse/util.py +37 -0
- amazon_subscriptions/serialize.py +29 -0
- amazon_subscriptions-1.0.1.dist-info/METADATA +314 -0
- amazon_subscriptions-1.0.1.dist-info/RECORD +24 -0
- amazon_subscriptions-1.0.1.dist-info/WHEEL +4 -0
- amazon_subscriptions-1.0.1.dist-info/entry_points.txt +2 -0
- amazon_subscriptions-1.0.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Parse the delivery cards of the landing page and the delivery pages."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from datetime import date, datetime, timedelta, timezone
|
|
7
|
+
from decimal import Decimal
|
|
8
|
+
from urllib.parse import parse_qs, urlparse
|
|
9
|
+
|
|
10
|
+
from bs4 import Tag
|
|
11
|
+
|
|
12
|
+
from amazon_subscriptions import dates
|
|
13
|
+
from amazon_subscriptions.exceptions import PageStructureError
|
|
14
|
+
from amazon_subscriptions.locales.base import SubscriptionLocale
|
|
15
|
+
from amazon_subscriptions.models import DeliveryItem, DiscountTier, ParseError, UpcomingDelivery
|
|
16
|
+
from amazon_subscriptions.parse import selectors
|
|
17
|
+
from amazon_subscriptions.parse.util import Html, attr_of, check_page, soup_of, text_of
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse_delivery_cards(
|
|
23
|
+
html: Html, locale: SubscriptionLocale, page: str | None = None, fragment: bool = False
|
|
24
|
+
) -> list[UpcomingDelivery]:
|
|
25
|
+
"""Parse the delivery cards of the landing page, without items. Their ``url`` leads to the delivery page.
|
|
26
|
+
|
|
27
|
+
:param fragment: ``True`` for a fragment returned by the pagination of the deliveries widget. It is empty
|
|
28
|
+
(``<div></div>``) if there are no further deliveries, else it must have cards. The landing page must have the
|
|
29
|
+
widget, but may have no cards.
|
|
30
|
+
:raises PageStructureError: If the landing page has no deliveries widget, a fragment with content has no cards,
|
|
31
|
+
or a card has no date.
|
|
32
|
+
"""
|
|
33
|
+
soup = soup_of(html)
|
|
34
|
+
check_page(soup, page)
|
|
35
|
+
cards = soup.select(selectors.DELIVERY_CARD)
|
|
36
|
+
if fragment and not cards and not soup.get_text(strip=True) and not soup.find(["img", "a", "span"]):
|
|
37
|
+
return []
|
|
38
|
+
if fragment and not cards:
|
|
39
|
+
raise PageStructureError(
|
|
40
|
+
f"No delivery cards found ({selectors.DELIVERY_CARD!r})", page, selector=selectors.DELIVERY_CARD
|
|
41
|
+
)
|
|
42
|
+
if not fragment and soup.select_one(selectors.DELIVERIES_WIDGET) is None:
|
|
43
|
+
raise PageStructureError(
|
|
44
|
+
f"No deliveries found ({selectors.DELIVERIES_WIDGET!r})", page, selector=selectors.DELIVERIES_WIDGET
|
|
45
|
+
)
|
|
46
|
+
return [_parse_card(card, locale, page) for card in cards]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parse_deliveries_next_url(html: Html) -> str | None:
|
|
50
|
+
"""The URL of the next page of the deliveries widget, if there is one."""
|
|
51
|
+
return attr_of(soup_of(html).select_one(selectors.DELIVERIES_NEXT_PAGE), "data-next-url")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse_card(card: Tag, locale: SubscriptionLocale, page: str | None) -> UpcomingDelivery:
|
|
55
|
+
url = attr_of(card.select_one(selectors.DELIVERY_CARD_LINK), "href")
|
|
56
|
+
query = parse_qs(urlparse(url or "").query)
|
|
57
|
+
epoch_ms = _delivery_date_param(url)
|
|
58
|
+
arrival = deadline = None
|
|
59
|
+
for text in (text_of(tag) for tag in card.select("span")):
|
|
60
|
+
if text and text.startswith(locale.ARRIVAL_PREFIX):
|
|
61
|
+
arrival = locale.parse_date_parts(text)
|
|
62
|
+
elif text and text.startswith(locale.DEADLINE_PREFIX):
|
|
63
|
+
deadline = locale.parse_date_parts(text)
|
|
64
|
+
if arrival is None or not epoch_ms.isdigit():
|
|
65
|
+
raise PageStructureError(f"Delivery card without arrival date or link: {url!r}", page)
|
|
66
|
+
|
|
67
|
+
# The timestamp is only used to find the year of the rendered date.
|
|
68
|
+
reference = _date_of_epoch_ms(epoch_ms)
|
|
69
|
+
assert reference is not None
|
|
70
|
+
delivery_date = dates.resolve(arrival, lambda d, m: dates.nearest(d, m, reference))
|
|
71
|
+
if delivery_date is None:
|
|
72
|
+
raise PageStructureError(f"Delivery card with invalid arrival date: {url!r}", page)
|
|
73
|
+
|
|
74
|
+
images = len(card.select(selectors.DELIVERY_CARD_IMAGE))
|
|
75
|
+
more = re.sub(r"\D", "", text_of(card.select_one(selectors.DELIVERY_CARD_MORE_ITEMS)) or "")
|
|
76
|
+
return UpcomingDelivery(
|
|
77
|
+
date=delivery_date,
|
|
78
|
+
change_deadline=dates.resolve(deadline, lambda d, m: dates.last_on_or_before(d, m, delivery_date)),
|
|
79
|
+
discount_tier=DiscountTier(item_count=images + int(more or 0) if images else None),
|
|
80
|
+
currency=locale.currency,
|
|
81
|
+
delivery_bundle_id=next(iter(query.get("deliveryBundleId", [])), None),
|
|
82
|
+
url=url,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse_delivery_page(html: Html, locale: SubscriptionLocale, page: str | None = None) -> UpcomingDelivery:
|
|
87
|
+
"""Parse a delivery page with its items.
|
|
88
|
+
|
|
89
|
+
Items are not linked to subscriptions yet, see :func:`amazon_subscriptions.parse.matching.match_deliveries`.
|
|
90
|
+
|
|
91
|
+
If the page shows no delivery date, it is taken from the ``deliveryDate`` of the page URL ``page``, and the missing
|
|
92
|
+
date is noted in ``parse_errors`` of the delivery.
|
|
93
|
+
|
|
94
|
+
:raises PageStructureError: If neither the page nor its URL has a delivery date, or the page has no items.
|
|
95
|
+
"""
|
|
96
|
+
soup = soup_of(html)
|
|
97
|
+
check_page(soup, page)
|
|
98
|
+
parse_errors = []
|
|
99
|
+
delivery_date = _parse_delivery_date(text_of(soup.select_one(selectors.DELIVERY_DATE)))
|
|
100
|
+
if delivery_date is None:
|
|
101
|
+
delivery_date = _date_of_epoch_ms(_delivery_date_param(page))
|
|
102
|
+
if delivery_date is None:
|
|
103
|
+
raise PageStructureError(
|
|
104
|
+
f"No delivery date found ({selectors.DELIVERY_DATE!r})", page, selector=selectors.DELIVERY_DATE
|
|
105
|
+
)
|
|
106
|
+
logger.debug(f"The delivery page shows no delivery date, so it was taken from its URL: {delivery_date}")
|
|
107
|
+
parse_errors.append(
|
|
108
|
+
ParseError(
|
|
109
|
+
url=page,
|
|
110
|
+
message="No delivery date found, it was taken from the URL",
|
|
111
|
+
selector=selectors.DELIVERY_DATE,
|
|
112
|
+
)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
item_tags = soup.select(selectors.DELIVERY_ITEM)
|
|
116
|
+
if not item_tags:
|
|
117
|
+
raise PageStructureError(
|
|
118
|
+
f"No delivery items found ({selectors.DELIVERY_ITEM!r})", page, selector=selectors.DELIVERY_ITEM
|
|
119
|
+
)
|
|
120
|
+
items = [_parse_item(tag, locale) for tag in item_tags]
|
|
121
|
+
|
|
122
|
+
item_count = locale.parse_item_count(text_of(soup.select_one(selectors.DELIVERY_ITEM_COUNT)))
|
|
123
|
+
if item_count is not None and item_count != len(items):
|
|
124
|
+
logger.warning(f"Delivery of {delivery_date} states {item_count} items, but {len(items)} were found.")
|
|
125
|
+
savings_text = text_of(soup.select_one(selectors.DELIVERY_SAVINGS))
|
|
126
|
+
deadline = locale.parse_date_parts(text_of(soup.select_one(selectors.DELIVERY_EDIT_DEADLINE)))
|
|
127
|
+
|
|
128
|
+
prices = [item.price for item in items]
|
|
129
|
+
return UpcomingDelivery(
|
|
130
|
+
date=delivery_date,
|
|
131
|
+
change_deadline=dates.resolve(deadline, lambda d, m: dates.last_on_or_before(d, m, delivery_date)),
|
|
132
|
+
items=items,
|
|
133
|
+
discount_tier=DiscountTier(
|
|
134
|
+
item_count=item_count if item_count is not None else len(items),
|
|
135
|
+
max_discount_unlocked=(
|
|
136
|
+
locale.MAX_DISCOUNT_UNLOCKED_TEXT.lower() in savings_text.lower() if savings_text else None
|
|
137
|
+
),
|
|
138
|
+
savings=locale.parse_savings(savings_text),
|
|
139
|
+
),
|
|
140
|
+
total=sum((p for p in prices if p is not None), Decimal("0")) if None not in prices else None,
|
|
141
|
+
currency=locale.currency,
|
|
142
|
+
parse_errors=parse_errors,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _delivery_date_param(url: str | None) -> str:
|
|
147
|
+
"""The ``deliveryDate`` of a delivery link, a timestamp in milliseconds, or ``""``."""
|
|
148
|
+
return (parse_qs(urlparse(url or "").query).get("deliveryDate") or [""])[0]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _date_of_epoch_ms(epoch_ms: str) -> date | None:
|
|
152
|
+
"""The date of a timestamp in milliseconds that is midnight in the storefront's time zone.
|
|
153
|
+
|
|
154
|
+
The time zone is not needed: midnight at a UTC offset between -12 h (exclusive) and +12 h, which covers every
|
|
155
|
+
Amazon storefront, is between noon of the day before and noon of the day in UTC. So 12 hours later it is on
|
|
156
|
+
the day in UTC.
|
|
157
|
+
"""
|
|
158
|
+
if not epoch_ms.isdigit():
|
|
159
|
+
return None
|
|
160
|
+
return (datetime.fromtimestamp(int(epoch_ms) / 1000, tz=timezone.utc) + timedelta(hours=12)).date()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _parse_delivery_date(text: str | None) -> date | None:
|
|
164
|
+
"""The date of ``{"value":{"dateTime":"2026-10-01T00:00:00.000+02:00","zoneId":"Europe/Paris"}}``."""
|
|
165
|
+
try:
|
|
166
|
+
value = json.loads(text or "")["value"]["dateTime"]
|
|
167
|
+
# The local date of the storefront, not converted to UTC
|
|
168
|
+
return date.fromisoformat(value[:10])
|
|
169
|
+
except (ValueError, KeyError, TypeError):
|
|
170
|
+
logger.debug(f"Delivery date {text!r} was not recognized, so it was not parsed.")
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _item_container(tag: Tag) -> Tag:
|
|
175
|
+
"""The largest element around an item tile that holds no other item, which also holds the item's alert."""
|
|
176
|
+
container = tag
|
|
177
|
+
while container.parent is not None and len(container.parent.select(selectors.DELIVERY_ITEM)) == 1:
|
|
178
|
+
container = container.parent
|
|
179
|
+
return container
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _parse_item(tag: Tag, locale: SubscriptionLocale) -> DeliveryItem:
|
|
183
|
+
price_tag = tag.select_one(selectors.DELIVERY_ITEM_PRICE)
|
|
184
|
+
price_text = None
|
|
185
|
+
if price_tag is not None:
|
|
186
|
+
labelled = price_tag.select_one("[aria-label]")
|
|
187
|
+
price_text = attr_of(labelled, "aria-label") or text_of(price_tag, separator="")
|
|
188
|
+
alert = text_of(_item_container(tag).select_one(selectors.DELIVERY_ITEM_ALERT))
|
|
189
|
+
quantity_interval_text = text_of(tag.select_one(selectors.DELIVERY_ITEM_QUANTITY_INTERVAL))
|
|
190
|
+
quantity_interval = locale.parse_quantity_interval(quantity_interval_text)
|
|
191
|
+
return DeliveryItem(
|
|
192
|
+
title=text_of(tag.select_one(selectors.DELIVERY_ITEM_TITLE)) or "",
|
|
193
|
+
quantity=quantity_interval[0] if quantity_interval else None,
|
|
194
|
+
interval=quantity_interval[1] if quantity_interval else None,
|
|
195
|
+
price=locale.parse_amount(price_text),
|
|
196
|
+
discount_percent=locale.parse_discount_percent(text_of(tag.select_one(selectors.DELIVERY_ITEM_DISCOUNT))),
|
|
197
|
+
alert_text=alert,
|
|
198
|
+
substitute=bool(alert and locale.is_substitute_alert(alert)),
|
|
199
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Parse the detail sheet of a subscription, which the landing page opens for a subscription tile."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from urllib.parse import parse_qs, urlparse
|
|
5
|
+
|
|
6
|
+
from amazon_subscriptions.parse import selectors
|
|
7
|
+
from amazon_subscriptions.parse.util import Html, attr_of, check_page, soup_of, text_of
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class BackupItem:
|
|
12
|
+
"""The backup product of a subscription, sent if the subscribed product is not available."""
|
|
13
|
+
|
|
14
|
+
asin: str
|
|
15
|
+
title: str | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def parse_subscription_detail(html: Html, page: str | None = None) -> BackupItem | None:
|
|
19
|
+
"""The backup product of a subscription's detail sheet, ``None`` if it has none."""
|
|
20
|
+
soup = soup_of(html)
|
|
21
|
+
check_page(soup, page)
|
|
22
|
+
link = soup.select_one(selectors.SUBSCRIPTION_DETAIL_BACKUP_LINK)
|
|
23
|
+
query = parse_qs(urlparse(attr_of(link, "href") or "").query)
|
|
24
|
+
asin = next(iter(query.get("ASIN", [])), None)
|
|
25
|
+
if link is None or not asin:
|
|
26
|
+
return None
|
|
27
|
+
return BackupItem(asin=asin, title=text_of(link.select_one(selectors.SUBSCRIPTION_DETAIL_BACKUP_TITLE)))
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Link delivery items to subscriptions.
|
|
2
|
+
|
|
3
|
+
Delivery pages show neither ASIN nor subscription id, and the product images differ from the ones of the
|
|
4
|
+
subscription tiles. The only link is the title, which on delivery pages is often a shortened version of the
|
|
5
|
+
subscription's title. So an item is matched to a subscription by, in this order:
|
|
6
|
+
|
|
7
|
+
1. equal title,
|
|
8
|
+
2. the item's title being the first words of the subscription's title,
|
|
9
|
+
3. elimination: a backup product (see :attr:`DeliveryItem.substitute`) is matched to the only subscription due on
|
|
10
|
+
the delivery date that has no item yet, if quantity and interval are the same.
|
|
11
|
+
|
|
12
|
+
Each step only matches if exactly one subscription qualifies. Items without a unique match keep
|
|
13
|
+
``subscription_id=None``. Titles are never matched by similarity, since a wrong match is worse than none.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
from collections.abc import Callable, Sequence
|
|
18
|
+
|
|
19
|
+
from amazon_subscriptions.models import DeliveryItem, Subscription, SubscriptionStatus, UpcomingDelivery
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _words(title: str | None) -> list[str]:
|
|
25
|
+
return (title or "").rstrip(" .…").casefold().split()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def titles_match(a: str | None, b: str | None) -> bool:
|
|
29
|
+
"""``True`` if both titles are equal, or one is the first words of the other."""
|
|
30
|
+
shorter, longer = sorted((_words(a), _words(b)), key=len)
|
|
31
|
+
return bool(shorter) and longer[: len(shorter)] == shorter
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_prefix(item: DeliveryItem, subscription: Subscription) -> bool:
|
|
35
|
+
item_words, subscription_words = _words(item.title), _words(subscription.title)
|
|
36
|
+
return 0 < len(item_words) < len(subscription_words) and subscription_words[: len(item_words)] == item_words
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _is_equal(item: DeliveryItem, subscription: Subscription) -> bool:
|
|
40
|
+
return bool(_words(item.title)) and _words(item.title) == _words(subscription.title)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _link(item: DeliveryItem, subscription: Subscription) -> None:
|
|
44
|
+
item.subscription_id = subscription.subscription_id
|
|
45
|
+
item.asin = subscription.asin
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def match_delivery(delivery: UpcomingDelivery, subscriptions: Sequence[Subscription]) -> None:
|
|
49
|
+
"""Link the items of a delivery to their subscriptions, see the module documentation."""
|
|
50
|
+
unmatched = list(subscriptions)
|
|
51
|
+
open_items = [item for item in delivery.items if item.subscription_id is None]
|
|
52
|
+
|
|
53
|
+
rule: Callable[[DeliveryItem, Subscription], bool]
|
|
54
|
+
for rule in (_is_equal, _is_prefix):
|
|
55
|
+
for item in list(open_items):
|
|
56
|
+
candidates = [s for s in unmatched if rule(item, s)]
|
|
57
|
+
if len(candidates) == 1:
|
|
58
|
+
_link(item, candidates[0])
|
|
59
|
+
unmatched.remove(candidates[0])
|
|
60
|
+
open_items.remove(item)
|
|
61
|
+
elif len(candidates) > 1:
|
|
62
|
+
logger.warning(f"Item {item.title!r} of {delivery.date} matches {len(candidates)} subscriptions.")
|
|
63
|
+
|
|
64
|
+
substitutes = [item for item in open_items if item.substitute]
|
|
65
|
+
due = [s for s in unmatched if s.next_delivery_date == delivery.date]
|
|
66
|
+
if len(substitutes) == 1 and len(due) == 1:
|
|
67
|
+
item, subscription = substitutes[0], due[0]
|
|
68
|
+
if (item.quantity, item.interval) == (subscription.quantity, subscription.interval):
|
|
69
|
+
_link(item, subscription)
|
|
70
|
+
open_items.remove(item)
|
|
71
|
+
|
|
72
|
+
for item in open_items:
|
|
73
|
+
logger.warning(f"Item {item.title!r} of {delivery.date} matches no subscription.")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def apply_next_delivery(subscriptions: Sequence[Subscription], deliveries: Sequence[UpcomingDelivery]) -> None:
|
|
77
|
+
"""Copy price and discount of each subscription's next delivery to the subscription.
|
|
78
|
+
|
|
79
|
+
A subscription whose next delivery sends a backup product gets the status ``unavailable`` and the item's alert as
|
|
80
|
+
status text, but no price, since the price is the one of the backup product.
|
|
81
|
+
"""
|
|
82
|
+
by_id = {s.subscription_id: s for s in subscriptions}
|
|
83
|
+
for delivery in deliveries:
|
|
84
|
+
for item in delivery.items:
|
|
85
|
+
subscription = by_id.get(item.subscription_id or "")
|
|
86
|
+
if subscription is None or subscription.next_delivery_date != delivery.date:
|
|
87
|
+
continue
|
|
88
|
+
if item.substitute:
|
|
89
|
+
subscription.status = SubscriptionStatus.UNAVAILABLE
|
|
90
|
+
subscription.raw_status_text = item.alert_text
|
|
91
|
+
continue
|
|
92
|
+
subscription.subscription_price = item.price
|
|
93
|
+
subscription.discount_percent = item.discount_percent
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def match_deliveries(subscriptions: Sequence[Subscription], deliveries: Sequence[UpcomingDelivery]) -> None:
|
|
97
|
+
"""Link the items of all deliveries to their subscriptions and copy the next delivery's data to them."""
|
|
98
|
+
for delivery in deliveries:
|
|
99
|
+
match_delivery(delivery, subscriptions)
|
|
100
|
+
apply_next_delivery(subscriptions, deliveries)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Parse the prices of a product page (/dp/<ASIN>), which the Subscribe & Save pages do not show.
|
|
2
|
+
|
|
3
|
+
A product page shows prices of many products: other variants, sponsored products and recommendations. Prices are only
|
|
4
|
+
read from blocks that belong to the product of the page (by its ASIN) and, for list prices, to the offer they are shown
|
|
5
|
+
for.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from decimal import Decimal
|
|
12
|
+
|
|
13
|
+
from bs4 import BeautifulSoup, Tag
|
|
14
|
+
|
|
15
|
+
from amazon_subscriptions.locales.base import SubscriptionLocale
|
|
16
|
+
from amazon_subscriptions.models import AlternativeOffer, ListPriceSource, ListPriceType
|
|
17
|
+
from amazon_subscriptions.parse import selectors
|
|
18
|
+
from amazon_subscriptions.parse.util import Html, attr_of, check_page, soup_of, text_of
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
#: ASINs in the links of a comparison widget entry
|
|
23
|
+
_LINK_ASIN_RE = re.compile(r"/(?:dp|gp/product|product-reviews)/([A-Z0-9]{10})\b|[?&]pd_rd_i=([A-Z0-9]{10})\b")
|
|
24
|
+
_ITEM_ID_PREFIX = "amzn1.asin."
|
|
25
|
+
_SELLER_ID_RE = re.compile(r"[?&]seller=([A-Z0-9]+)")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class ProductPrices:
|
|
30
|
+
"""The prices of a product page. All ``None`` if the product is not available."""
|
|
31
|
+
|
|
32
|
+
#: Price of a one-time purchase in the buybox.
|
|
33
|
+
price: Decimal | None = None
|
|
34
|
+
#: List price, if the page shows one for this product.
|
|
35
|
+
list_price: Decimal | None = None
|
|
36
|
+
#: What ``list_price`` is, by its label.
|
|
37
|
+
list_price_type: ListPriceType | None = None
|
|
38
|
+
#: The block ``list_price`` was read from.
|
|
39
|
+
list_price_source: ListPriceSource | None = None
|
|
40
|
+
#: Price per unit of measure of ``price``.
|
|
41
|
+
unit_price: Decimal | None = None
|
|
42
|
+
#: Unit of measure of ``unit_price``, e.g. ``l`` or ``count``.
|
|
43
|
+
unit_price_unit: str | None = None
|
|
44
|
+
#: The cheapest other offer of the product next to the buybox offer.
|
|
45
|
+
alternative_offer: AlternativeOffer | None = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class _ListPrice:
|
|
50
|
+
amount: Decimal
|
|
51
|
+
type: ListPriceType
|
|
52
|
+
source: ListPriceSource
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_product_page(html: Html, locale: SubscriptionLocale, page: str | None = None) -> ProductPrices:
|
|
56
|
+
"""The prices of the product of a product page: buybox price, list price, unit price and alternative offer."""
|
|
57
|
+
soup = soup_of(html)
|
|
58
|
+
check_page(soup, page)
|
|
59
|
+
offers = soup.select(selectors.PRODUCT_OFFER)
|
|
60
|
+
buybox = next((offer for offer in offers if not _is_alternative(offer, locale)), None)
|
|
61
|
+
price = _offer_price(buybox, locale)
|
|
62
|
+
if buybox is None or price is None:
|
|
63
|
+
availability = text_of(soup.select_one(selectors.PRODUCT_AVAILABILITY))
|
|
64
|
+
logger.debug(f"No price on the product page {page or ''}: {availability or 'no availability shown'}")
|
|
65
|
+
return ProductPrices()
|
|
66
|
+
|
|
67
|
+
asin = attr_of(buybox, "data-csa-c-asin")
|
|
68
|
+
alternatives = [offer for offer in offers if _is_alternative(offer, locale) and _is_of(offer, asin)]
|
|
69
|
+
unit_price = locale.parse_unit_price(text_of(buybox.select_one(selectors.PRODUCT_UNIT_PRICE)))
|
|
70
|
+
list_price = (
|
|
71
|
+
_display_list_price(soup, offers, buybox, locale, ListPriceSource.BUYBOX)
|
|
72
|
+
or _widget_list_price(soup, asin, locale)
|
|
73
|
+
or next(
|
|
74
|
+
(
|
|
75
|
+
found
|
|
76
|
+
for offer in alternatives
|
|
77
|
+
if (found := _display_list_price(soup, offers, offer, locale, ListPriceSource.ALTERNATIVE_OFFER))
|
|
78
|
+
),
|
|
79
|
+
None,
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
return ProductPrices(
|
|
84
|
+
price=price,
|
|
85
|
+
list_price=list_price.amount if list_price else None,
|
|
86
|
+
list_price_type=list_price.type if list_price else None,
|
|
87
|
+
list_price_source=list_price.source if list_price else None,
|
|
88
|
+
unit_price=unit_price[0] if unit_price else None,
|
|
89
|
+
unit_price_unit=unit_price[1] if unit_price else None,
|
|
90
|
+
alternative_offer=_cheapest_alternative(alternatives, locale),
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _offer_row(offer: Tag) -> Tag | None:
|
|
95
|
+
"""The buybox accordion row of an offer, ``None`` if the page has no accordion."""
|
|
96
|
+
return offer.find_parent(id=re.compile(r"^newAccordionRow"))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_alternative(offer: Tag, locale: SubscriptionLocale) -> bool:
|
|
100
|
+
row = _offer_row(offer)
|
|
101
|
+
return row is not None and locale.is_alternative_offers_caption(
|
|
102
|
+
text_of(row.select_one(selectors.PRODUCT_OFFER_CAPTION))
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _is_of(block: Tag, asin: str | None) -> bool:
|
|
107
|
+
"""Whether a block of the page is one of the product ``asin``. Blocks without ASIN belong to the page."""
|
|
108
|
+
block_asin = attr_of(block, "data-csa-c-asin")
|
|
109
|
+
return block_asin is None or block_asin == asin
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _offer_price(offer: Tag | None, locale: SubscriptionLocale) -> Decimal | None:
|
|
113
|
+
return locale.parse_amount(text_of(offer.select_one(selectors.PRODUCT_PRICE))) if offer else None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _display_list_price(
|
|
117
|
+
soup: BeautifulSoup, offers: list[Tag], offer: Tag, locale: SubscriptionLocale, source: ListPriceSource
|
|
118
|
+
) -> _ListPrice | None:
|
|
119
|
+
"""The list price in the price display of an offer. Of an alternative offer, only a recommended retail price.
|
|
120
|
+
|
|
121
|
+
The center column has one price display per offer, in the order of the offers, of which only the one of the
|
|
122
|
+
selected offer is visible. A display is only used if it shows the price of its offer.
|
|
123
|
+
"""
|
|
124
|
+
displays = soup.select(selectors.PRODUCT_PRICE_DISPLAY)
|
|
125
|
+
if len(displays) != len(offers):
|
|
126
|
+
logger.debug(f"{len(displays)} price displays for {len(offers)} offers, so no list price is read from them.")
|
|
127
|
+
return None
|
|
128
|
+
# Tags compare equal by their content, so the offer is found by identity
|
|
129
|
+
display = displays[next(i for i, candidate in enumerate(offers) if candidate is offer)]
|
|
130
|
+
shown = locale.parse_amount(text_of(display.select_one(selectors.PRODUCT_DISPLAY_PRICE), separator=""))
|
|
131
|
+
price = _offer_price(offer, locale)
|
|
132
|
+
if shown != price or not _is_of(display, attr_of(offer, "data-csa-c-asin")):
|
|
133
|
+
logger.debug(f"The price display shows {shown} instead of the offer's {price}, so it is not used.")
|
|
134
|
+
return None
|
|
135
|
+
for basis in display.select(selectors.PRODUCT_BASIS_PRICE):
|
|
136
|
+
list_price_type = locale.list_price_type_of(text_of(basis.select_one(selectors.PRODUCT_BASIS_PRICE_LABEL)))
|
|
137
|
+
amount = locale.parse_amount(text_of(basis.select_one(selectors.PRODUCT_BASIS_PRICE_VALUE)))
|
|
138
|
+
if list_price_type is None or amount is None:
|
|
139
|
+
continue
|
|
140
|
+
if source is ListPriceSource.BUYBOX or list_price_type is ListPriceType.RRP:
|
|
141
|
+
return _ListPrice(amount, list_price_type, source)
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _widget_list_price(soup: BeautifulSoup, asin: str | None, locale: SubscriptionLocale) -> _ListPrice | None:
|
|
146
|
+
"""The recommended retail price of the entry of the product itself in a comparison widget.
|
|
147
|
+
|
|
148
|
+
Only the recommended retail price is read: the entry may show another offer than the buybox, and other list
|
|
149
|
+
prices belong to that offer.
|
|
150
|
+
"""
|
|
151
|
+
if asin is None:
|
|
152
|
+
return None
|
|
153
|
+
for label in soup.select(f"{selectors.PRODUCT_WIDGET} {selectors.PRODUCT_WIDGET_LABEL}"):
|
|
154
|
+
if not locale.is_this_item_label(text_of(label)):
|
|
155
|
+
continue
|
|
156
|
+
entry = label.find_parent(class_=selectors.PRODUCT_WIDGET_ENTRY)
|
|
157
|
+
if entry is None or _widget_entry_asins(label, entry) != {asin}:
|
|
158
|
+
continue
|
|
159
|
+
for strike in entry.select(selectors.PRODUCT_WIDGET_STRIKE_PRICE):
|
|
160
|
+
value = text_of(strike) or ""
|
|
161
|
+
if locale.list_price_type_of((text_of(strike.parent) or "").replace(value, "")) is ListPriceType.RRP:
|
|
162
|
+
amount = locale.parse_amount(value)
|
|
163
|
+
if amount is not None:
|
|
164
|
+
return _ListPrice(amount, ListPriceType.RRP, ListPriceSource.WIDGET)
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _widget_entry_asins(label: Tag, entry: Tag) -> set[str]:
|
|
169
|
+
"""The ASINs of a widget entry: of its item ID, or else of its links."""
|
|
170
|
+
for parent in label.parents:
|
|
171
|
+
item_id = attr_of(parent, "data-csa-c-item-id")
|
|
172
|
+
if item_id and item_id.startswith(_ITEM_ID_PREFIX):
|
|
173
|
+
return {item_id.removeprefix(_ITEM_ID_PREFIX)}
|
|
174
|
+
if attr_of(parent, "data-feature-name"):
|
|
175
|
+
break
|
|
176
|
+
return {
|
|
177
|
+
match.group(1) or match.group(2)
|
|
178
|
+
for link in entry.select("a[href]")
|
|
179
|
+
for match in _LINK_ASIN_RE.finditer(attr_of(link, "href") or "")
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _cheapest_alternative(alternatives: list[Tag], locale: SubscriptionLocale) -> AlternativeOffer | None:
|
|
184
|
+
found = []
|
|
185
|
+
for offer in alternatives:
|
|
186
|
+
price = _offer_price(offer, locale)
|
|
187
|
+
if price is None:
|
|
188
|
+
continue
|
|
189
|
+
unit_price = locale.parse_unit_price(text_of(offer.select_one(selectors.PRODUCT_UNIT_PRICE)))
|
|
190
|
+
row = _offer_row(offer)
|
|
191
|
+
seller_link = row.select_one(selectors.PRODUCT_OFFER_SELLER_LINK) if row else None
|
|
192
|
+
seller_id = _SELLER_ID_RE.search(attr_of(seller_link, "href") or "")
|
|
193
|
+
found.append(
|
|
194
|
+
AlternativeOffer(
|
|
195
|
+
price=price,
|
|
196
|
+
unit_price=unit_price[0] if unit_price else None,
|
|
197
|
+
unit_price_unit=unit_price[1] if unit_price else None,
|
|
198
|
+
seller=text_of(row.select_one(selectors.PRODUCT_OFFER_SELLER)) if row else None,
|
|
199
|
+
seller_id=seller_id.group(1) if seller_id else None,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
return min(found, key=lambda offer: offer.price, default=None)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""CSS selectors of the Subscribe & Save pages.
|
|
2
|
+
|
|
3
|
+
Classes with hashes (e.g. ``_myd-hub-..._style_tile__1pKIG``) change with every deployment, so only ids, ``data-``
|
|
4
|
+
attributes and stable class names are used.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
# Landing page (/auto-deliveries), and the fragments of its paginated subscriptions grid
|
|
8
|
+
SUBSCRIPTIONS_WIDGET = "[data-acp-path*='subscriptions-card']"
|
|
9
|
+
SUBSCRIPTIONS_GRID = "#subscriptionsDesktopGridLayout"
|
|
10
|
+
SUBSCRIPTION_TILE = "[data-edit-url*='subscriptionId=']"
|
|
11
|
+
SUBSCRIPTION_TILE_TITLE = ".a-truncate-full"
|
|
12
|
+
SUBSCRIPTION_TILE_IMAGE = "img"
|
|
13
|
+
SUBSCRIPTIONS_NEXT_PAGE = "[data-next-url*='/replenishment/subscriptions']"
|
|
14
|
+
|
|
15
|
+
DELIVERIES_WIDGET = "[data-acp-path*='deliveries-card']"
|
|
16
|
+
DELIVERY_CARD = ".hub-delivery-card"
|
|
17
|
+
DELIVERY_CARD_LINK = "a[href*='deliveryDate=']"
|
|
18
|
+
DELIVERY_CARD_IMAGE = "img"
|
|
19
|
+
DELIVERY_CARD_MORE_ITEMS = "[class*='plusCount']"
|
|
20
|
+
DELIVERIES_NEXT_PAGE = "[data-next-url*='/replenishment/fulfillment/deliveries']"
|
|
21
|
+
|
|
22
|
+
# Delivery page (/auto-deliveries/?deliveryDate=...)
|
|
23
|
+
DELIVERY_DATE = "[data-testid='ddp-atd-delivery-date']"
|
|
24
|
+
DELIVERY_EDIT_DEADLINE = "[data-testid='delivery-edit-deadline-date-desktop']"
|
|
25
|
+
DELIVERY_ITEM_COUNT = "[data-testid='delivery-item-count-desktop']"
|
|
26
|
+
DELIVERY_SAVINGS = "[data-testid='savings-celebration-desktop']"
|
|
27
|
+
DELIVERY_ITEM = "[data-testid='desktop-subscription-tile']"
|
|
28
|
+
DELIVERY_ITEM_TITLE = "[data-testid='product-title']"
|
|
29
|
+
DELIVERY_ITEM_QUANTITY_INTERVAL = "[data-testid='product-qty-frequency']"
|
|
30
|
+
DELIVERY_ITEM_PRICE = "[data-testid='product-price']"
|
|
31
|
+
DELIVERY_ITEM_DISCOUNT = "[data-testid='product-savings']"
|
|
32
|
+
DELIVERY_ITEM_ALERT = "[data-testid='subscription-item-alert']"
|
|
33
|
+
|
|
34
|
+
# Detail sheet of a subscription (/auto-deliveries/ajax/subscription/?subscriptionId=...)
|
|
35
|
+
SUBSCRIPTION_DETAIL_BACKUP_LINK = "a[href*='/backupItem'][href*='ASIN=']"
|
|
36
|
+
SUBSCRIPTION_DETAIL_BACKUP_TITLE = ".actionDetail"
|
|
37
|
+
|
|
38
|
+
# Product page (/dp/<ASIN>). Each offer (buybox, alternative offers) is a row of the buybox accordion with the price of
|
|
39
|
+
# a one-time purchase. The center column has one price display per row, in the same order, with the list price.
|
|
40
|
+
PRODUCT_TITLE = "#productTitle"
|
|
41
|
+
PRODUCT_AVAILABILITY = "#availability"
|
|
42
|
+
PRODUCT_OFFER = "#corePrice_feature_div"
|
|
43
|
+
PRODUCT_OFFER_CAPTION = "#newAccordionCaption_feature_div"
|
|
44
|
+
# The seller name as the row shows it. Signed in, #sellerProfileTriggerId is a link labeled with a help text instead.
|
|
45
|
+
PRODUCT_OFFER_SELLER = (
|
|
46
|
+
"[offer-display-feature-name='desktop-merchant-info-collapsed'] .offer-display-feature-text-message"
|
|
47
|
+
)
|
|
48
|
+
PRODUCT_OFFER_SELLER_LINK = "#sellerProfileTriggerId[href*='seller=']"
|
|
49
|
+
PRODUCT_PRICE = ".apex-pricetopay-value .a-offscreen"
|
|
50
|
+
PRODUCT_UNIT_PRICE = ".apex-priceperunit-accessibility-label"
|
|
51
|
+
PRODUCT_PRICE_DISPLAY = "#corePriceDisplay_desktop_feature_div"
|
|
52
|
+
PRODUCT_DISPLAY_PRICE = ".apex-pricetopay-value [aria-hidden]"
|
|
53
|
+
PRODUCT_BASIS_PRICE = ".basisPrice"
|
|
54
|
+
PRODUCT_BASIS_PRICE_LABEL = ".apex-basisprice-label"
|
|
55
|
+
PRODUCT_BASIS_PRICE_VALUE = ".apex-basisprice-value .a-offscreen"
|
|
56
|
+
# Comparison widgets (e.g. "Compare with similar items"), whose entry of the product itself is labeled
|
|
57
|
+
PRODUCT_WIDGET = "[data-feature-name^='sims']"
|
|
58
|
+
PRODUCT_WIDGET_LABEL = ".a-text-bold"
|
|
59
|
+
PRODUCT_WIDGET_ENTRY = "a-cardui"
|
|
60
|
+
PRODUCT_WIDGET_STRIKE_PRICE = ".a-text-strike"
|
|
61
|
+
|
|
62
|
+
# Pages that are not the requested one
|
|
63
|
+
CAPTCHA = ["form[action*='validateCaptcha']", "#captchacharacters", "img[src*='/captcha/']"]
|
|
64
|
+
SIGN_IN = ["form[name='signIn']", "#ap_email", "#ap_password", "#auth-mfa-otpcode"]
|