untappd-scraper 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- untappd_scraper-0.4.0/.gitignore +31 -0
- untappd_scraper-0.4.0/.gitlab-ci.yml +47 -0
- untappd_scraper-0.4.0/DESIGN.md +345 -0
- untappd_scraper-0.4.0/Dockerfile +6 -0
- untappd_scraper-0.4.0/LICENSE +21 -0
- untappd_scraper-0.4.0/PKG-INFO +124 -0
- untappd_scraper-0.4.0/README.md +75 -0
- untappd_scraper-0.4.0/pyproject.toml +79 -0
- untappd_scraper-0.4.0/query_user.py +16 -0
- untappd_scraper-0.4.0/ruff.toml +50 -0
- untappd_scraper-0.4.0/tests/__init__.py +0 -0
- untappd_scraper-0.4.0/tests/conftest.py +239 -0
- untappd_scraper-0.4.0/tests/html/beer.html +2499 -0
- untappd_scraper-0.4.0/tests/html/fridge-list.html +1821 -0
- untappd_scraper-0.4.0/tests/html/user-beer-history.html +1816 -0
- untappd_scraper-0.4.0/tests/html/user-list-1.html +1821 -0
- untappd_scraper-0.4.0/tests/html/user-list-2.html +1816 -0
- untappd_scraper-0.4.0/tests/html/user-venue-history.html +1960 -0
- untappd_scraper-0.4.0/tests/html/user.html +1323 -0
- untappd_scraper-0.4.0/tests/html/userlists.html +852 -0
- untappd_scraper-0.4.0/tests/html/venue_unv.html +2341 -0
- untappd_scraper-0.4.0/tests/html/venue_ver.html +3246 -0
- untappd_scraper-0.4.0/tests/html/venue_ver_nest.html +2004 -0
- untappd_scraper-0.4.0/tests/html/wishlist.html +1358 -0
- untappd_scraper-0.4.0/tests/test_beer.py +36 -0
- untappd_scraper-0.4.0/tests/test_user.py +99 -0
- untappd_scraper-0.4.0/tests/test_user_beer_history.py +19 -0
- untappd_scraper-0.4.0/tests/test_user_lists.py +70 -0
- untappd_scraper-0.4.0/tests/test_user_utils.py +9 -0
- untappd_scraper-0.4.0/tests/test_user_venue_history.py +20 -0
- untappd_scraper-0.4.0/tests/test_venue.py +89 -0
- untappd_scraper-0.4.0/try_beer_details.py +10 -0
- untappd_scraper-0.4.0/try_user.py +39 -0
- untappd_scraper-0.4.0/try_user_activity.py +21 -0
- untappd_scraper-0.4.0/try_user_details.py +20 -0
- untappd_scraper-0.4.0/try_user_had.py +21 -0
- untappd_scraper-0.4.0/try_user_lists.py +31 -0
- untappd_scraper-0.4.0/try_user_recent_venues.py +21 -0
- untappd_scraper-0.4.0/try_venue.py +115 -0
- untappd_scraper-0.4.0/try_venue_activity.py +15 -0
- untappd_scraper-0.4.0/try_venue_details.py +12 -0
- untappd_scraper-0.4.0/try_venue_menus.py +46 -0
- untappd_scraper-0.4.0/untappd_scraper/__init__.py +1 -0
- untappd_scraper-0.4.0/untappd_scraper/beer.py +193 -0
- untappd_scraper-0.4.0/untappd_scraper/html_session.py +231 -0
- untappd_scraper-0.4.0/untappd_scraper/logging_config.py +78 -0
- untappd_scraper-0.4.0/untappd_scraper/logs.yml +59 -0
- untappd_scraper-0.4.0/untappd_scraper/py.typed +0 -0
- untappd_scraper-0.4.0/untappd_scraper/structs/__init__.py +0 -0
- untappd_scraper-0.4.0/untappd_scraper/structs/mixins.py +63 -0
- untappd_scraper-0.4.0/untappd_scraper/structs/other.py +27 -0
- untappd_scraper-0.4.0/untappd_scraper/structs/web.py +183 -0
- untappd_scraper-0.4.0/untappd_scraper/user.py +173 -0
- untappd_scraper-0.4.0/untappd_scraper/user_beer_history.py +112 -0
- untappd_scraper-0.4.0/untappd_scraper/user_lists.py +268 -0
- untappd_scraper-0.4.0/untappd_scraper/user_lists_details.py +117 -0
- untappd_scraper-0.4.0/untappd_scraper/user_utils.py +21 -0
- untappd_scraper-0.4.0/untappd_scraper/user_venue_history.py +67 -0
- untappd_scraper-0.4.0/untappd_scraper/venue.py +168 -0
- untappd_scraper-0.4.0/untappd_scraper/venue_menus.py +321 -0
- untappd_scraper-0.4.0/untappd_scraper/venue_utils.py +14 -0
- untappd_scraper-0.4.0/untappd_scraper/web.py +113 -0
- untappd_scraper-0.4.0/uv.lock +788 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# temp / uninteresting files
|
|
2
|
+
.python-version
|
|
3
|
+
.vscode/
|
|
4
|
+
.vstags
|
|
5
|
+
__pycache__
|
|
6
|
+
.mypy_cache
|
|
7
|
+
|
|
8
|
+
# distribution
|
|
9
|
+
dist/
|
|
10
|
+
pip/
|
|
11
|
+
pip.zip
|
|
12
|
+
|
|
13
|
+
# Sphinx
|
|
14
|
+
docs/_build
|
|
15
|
+
docs/_static
|
|
16
|
+
docs/_templates
|
|
17
|
+
|
|
18
|
+
# project
|
|
19
|
+
html.sqlite
|
|
20
|
+
*.log
|
|
21
|
+
a.html
|
|
22
|
+
b.html
|
|
23
|
+
conf2.py
|
|
24
|
+
fadel.json
|
|
25
|
+
u.json
|
|
26
|
+
|
|
27
|
+
.coverage
|
|
28
|
+
coverage.xml
|
|
29
|
+
.testmondata
|
|
30
|
+
cov.xml
|
|
31
|
+
.DS_Store
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
stages:
|
|
2
|
+
- test
|
|
3
|
+
|
|
4
|
+
variables:
|
|
5
|
+
PYTHONDONTWRITEBYTECODE: 1
|
|
6
|
+
PYTHONPATH: "./:./UntappdUniques:./tests"
|
|
7
|
+
TZ: Australia/Sydney
|
|
8
|
+
|
|
9
|
+
# UV
|
|
10
|
+
UV_VERSION: 0.6.3
|
|
11
|
+
PYTHON_VERSION: 3.13
|
|
12
|
+
BASE_LAYER: bookworm-slim
|
|
13
|
+
# GitLab CI creates a separate mountpoint for the build directory,
|
|
14
|
+
# so we need to copy instead of using hard links.
|
|
15
|
+
UV_LINK_MODE: copy
|
|
16
|
+
|
|
17
|
+
before_script:
|
|
18
|
+
- uname -a
|
|
19
|
+
- python --version
|
|
20
|
+
- uv --version
|
|
21
|
+
- set -u
|
|
22
|
+
- export UV_INDEX="https://__token__:${GITLAB_PYPI_PAT}@gitlab.com/api/v4/projects/23576087/packages/pypi/simple"
|
|
23
|
+
|
|
24
|
+
test-with-uv:
|
|
25
|
+
image: ghcr.io/astral-sh/uv:$UV_VERSION-python$PYTHON_VERSION-$BASE_LAYER
|
|
26
|
+
variables:
|
|
27
|
+
UV_CACHE_DIR: .uv-cache
|
|
28
|
+
cache:
|
|
29
|
+
- key:
|
|
30
|
+
files:
|
|
31
|
+
- uv.lock
|
|
32
|
+
- .venv
|
|
33
|
+
paths:
|
|
34
|
+
- $UV_CACHE_DIR
|
|
35
|
+
|
|
36
|
+
stage: test
|
|
37
|
+
script:
|
|
38
|
+
# - env | sort
|
|
39
|
+
# Your `uv` commands
|
|
40
|
+
- uv sync --group test
|
|
41
|
+
# - uv pip list
|
|
42
|
+
- uv run coverage run -m pytest # Run all tests
|
|
43
|
+
- uv run coverage xml
|
|
44
|
+
- uv run coverage report
|
|
45
|
+
# recommended to reduce cache size
|
|
46
|
+
- uv cache prune --ci
|
|
47
|
+
coverage: '/TOTAL.*\s+(\d+%)/'
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
# What is Scraped?
|
|
2
|
+
| type | url | web struct | note |
|
|
3
|
+
| ---------------- | ------------------------------ | ---------------------------------- | ---------------------------------------------- |
|
|
4
|
+
| beer details | beer/{beer_id} | WebBeerDetails |
|
|
5
|
+
| user details | user/{user_id} | WebUserDetails<br> WebActivityBeer | has activity too |
|
|
6
|
+
| user had beers | user/{user_id}/beers | WebUserHadBeer |
|
|
7
|
+
| user list names | user/{user_id}/lists | WebUserList |
|
|
8
|
+
| user list beers | user/{user_id}/lists/{list_id} | WebUserListBeer |
|
|
9
|
+
| user venues | user/{user_id}/venues | WebUserVenueHistory |
|
|
10
|
+
| venue activity | venue/{venue_id}/activity | WebActivityBeer | unverified venue on same page as venue details |
|
|
11
|
+
| venue details | venue/{venue_id} | WebVenueDetails |
|
|
12
|
+
| venue menu names | venue/{venue_id} | WebVenueMenu |
|
|
13
|
+
| venue menu beers | venue/{venue_id} | WebVenueMenuBeer | may be a different URL |
|
|
14
|
+
|
|
15
|
+
# Loop
|
|
16
|
+
```
|
|
17
|
+
# load menu beers
|
|
18
|
+
if has menu-header
|
|
19
|
+
if has menu-selector
|
|
20
|
+
for each menu-selector
|
|
21
|
+
load menu
|
|
22
|
+
for each menu-section
|
|
23
|
+
process menu
|
|
24
|
+
else
|
|
25
|
+
for each menu-section
|
|
26
|
+
process menu
|
|
27
|
+
|
|
28
|
+
# process menu (sub)
|
|
29
|
+
for each menu-section-header (collaroy packed beers)
|
|
30
|
+
process menu-section-header
|
|
31
|
+
for each menu-section-list
|
|
32
|
+
while more beer
|
|
33
|
+
for each beer in list
|
|
34
|
+
process beer
|
|
35
|
+
if show-more-section and still_want_more():
|
|
36
|
+
load next page
|
|
37
|
+
|
|
38
|
+
# load checkins
|
|
39
|
+
if has menu
|
|
40
|
+
load activity page
|
|
41
|
+
|
|
42
|
+
for each checkin
|
|
43
|
+
log beer, time, serving, rating
|
|
44
|
+
```
|
|
45
|
+
# HTML
|
|
46
|
+
## Menus
|
|
47
|
+
|
|
48
|
+
### examples
|
|
49
|
+
| venue | type |
|
|
50
|
+
| -------- | ---------------- |
|
|
51
|
+
| 4 pines | unofficial |
|
|
52
|
+
| winona | single |
|
|
53
|
+
| sweeneys | multi |
|
|
54
|
+
| collaroy | multi selectable |
|
|
55
|
+
|
|
56
|
+
### menu tags
|
|
57
|
+
```
|
|
58
|
+
menu-header
|
|
59
|
+
|
|
60
|
+
missing for 4 pines
|
|
61
|
+
-header menu-select for winona
|
|
62
|
+
menu-select for sweeneys
|
|
63
|
+
menu-select multi for collaroy
|
|
64
|
+
|
|
65
|
+
menu-selector for collaroy
|
|
66
|
+
<select class=‘menu-selector’>
|
|
67
|
+
<option value=“xxx”>menu name</option>
|
|
68
|
+
|
|
69
|
+
menu-section-header
|
|
70
|
+
<h4>menu name<span>xx</span></h4>
|
|
71
|
+
|
|
72
|
+
missing for 4 pines
|
|
73
|
+
single for winona
|
|
74
|
+
multiple for sweeneys
|
|
75
|
+
single for collaroy
|
|
76
|
+
|
|
77
|
+
menu-section-list
|
|
78
|
+
<ul><li><div class=“beer-info”>
|
|
79
|
+
<div class=“beer-label”><span data-update-at=“date”></span>
|
|
80
|
+
<div class=“beer-details”>
|
|
81
|
+
<h5><a href=“xx”>beer name</a><em>style</em></h5>
|
|
82
|
+
<h6><span><a href=“brewery”>brewery</a><div class=“caps small” data-rating=“3.949”></h6>
|
|
83
|
+
</ul>
|
|
84
|
+
<a class=“more show-more-section” data-section-id=“123” data-venue-id=“123” data-menu-id=“123”>
|
|
85
|
+
```
|
|
86
|
+
### activity tags
|
|
87
|
+
```html
|
|
88
|
+
<div class=“checkin”>
|
|
89
|
+
<div class=“top”>
|
|
90
|
+
<p class=“text”><a class=“user”>Bob</a> is drinking a/n <a>beer</a> by <a>brewery</a> at <a>venue</a></p>
|
|
91
|
+
<div class=“checkin-comment><div class="rating-serving">
|
|
92
|
+
<p class="serving"><span>Can</span></p>
|
|
93
|
+
<div class="caps " data-rating="4.25">
|
|
94
|
+
</div>
|
|
95
|
+
<div class="feedback">
|
|
96
|
+
<div class="bottom">
|
|
97
|
+
<a href="/user/Martinlife/checkin/991966052" class="time timezoner track-click" data-gregtime="Sun, 31 Jan 2021 05:35:45 +0000">31 Jan 21</a>
|
|
98
|
+
```
|
|
99
|
+
## 4 Pines (no menu)
|
|
100
|
+
|
|
101
|
+
# Menu systems
|
|
102
|
+
|
|
103
|
+
There are several different presentations of a venue's menu:
|
|
104
|
+
| type | example | id | url |
|
|
105
|
+
| ------------------------- | ------------------- | ------- | --------------------------------------------------- |
|
|
106
|
+
| unverified | 4 Pines brewpub | 14705 | https://untappd.com/v/4-pines-brewpub/14705 |
|
|
107
|
+
| single menu | Winona | 8931121 | https://untappd.com/v/winona/8931121 |
|
|
108
|
+
| multi menu, all displayed | Sweeneys | 107565 | https://untappd.com/v/hotel-sweeney-s/107565 |
|
|
109
|
+
| multi menu, selectable | Collaroy Beach Club | 99967 | https://untappd.com/v/the-beach-club-collaroy/99967 |
|
|
110
|
+
|
|
111
|
+
## unverified
|
|
112
|
+
Not much to do here.
|
|
113
|
+
1. scrape the checkins. Can get user's personal rating (but use global if available), and check in time easily
|
|
114
|
+
2. take note of "recent". < 1 week?
|
|
115
|
+
3. can't page, so just take the 20
|
|
116
|
+
## single menu
|
|
117
|
+
Some of these are huge. Winona has 554 on their menu but they aren't necessarily available - they just don't clean it up. So only scrape 100(?) or only since a certain date (available in HTML)
|
|
118
|
+
1. get beer list name
|
|
119
|
+
2. get beers on first page, capturing
|
|
120
|
+
- global rating
|
|
121
|
+
- name
|
|
122
|
+
- brewery
|
|
123
|
+
- style
|
|
124
|
+
- date updated in
|
|
125
|
+
```html
|
|
126
|
+
<span class="new-trigger" data-update-at="Mon, 30 Nov 2020 05:44:33 +0000" style="display: none;">New</span>
|
|
127
|
+
```
|
|
128
|
+
3. repeat by "pressing" the "Show More Beers". This actually is a request for XHR data
|
|
129
|
+
```bash
|
|
130
|
+
curl -XGET -H "x-requested-with: XMLHttpRequest" "https://untappd.com/venue/more_menu/8931121/15?section_id=178512830"
|
|
131
|
+
```
|
|
132
|
+
Subsequent pressing gets a url like `https://untappd.com/venue/more_menu/8931121/30?section_id=178512830`. '8931121' is the venue ID and stays the same. Note the 15 changed to a 30, which is the start number (then goes 45, 60, ...). Section ID 178512830 stays the same too and needs to be scraped from the "more" button (or the other tags nearby).
|
|
133
|
+
```html
|
|
134
|
+
<a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="178512830" data-venue-id="8931121" data-menu-id="50008188" style="display: block;">Show More Beers</a>
|
|
135
|
+
```
|
|
136
|
+
This returns a JSON string with HTML in it, eg,
|
|
137
|
+
```json
|
|
138
|
+
{"count":15,"view":"\t\t<li>\n\t\t\t<div class=\"beer-info \">\n\t\t\t <div class=...}
|
|
139
|
+
```
|
|
140
|
+
HTML is a bunch of `<li>` tags, eg,
|
|
141
|
+
```html
|
|
142
|
+
<li>
|
|
143
|
+
<div class="beer-info ">
|
|
144
|
+
<div class="beer-label">
|
|
145
|
+
<span class="new-trigger" data-update-at="Thu, 25 Feb 2021 08:52:44 +0000" style="display: none;">New</span>
|
|
146
|
+
</div>
|
|
147
|
+
<div class="beer-details">
|
|
148
|
+
<h5><a class="track-click" data-track="menu" data-href=":beer"
|
|
149
|
+
href="/b/garage-project-hazy-daze-amarillo-and-mosaic/4185296">Hazy Daze- Amarillo & Mosaic</a>
|
|
150
|
+
<em>Pale Ale - New England</em></h5>
|
|
151
|
+
<h6><span>5.8% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery"
|
|
152
|
+
href="/w/garage-project/14539">Garage Project</a>
|
|
153
|
+
• </span>
|
|
154
|
+
|
|
155
|
+
<div class="caps small" data-rating="3.946">
|
|
156
|
+
<div class="cap cap-100"></div>
|
|
157
|
+
<div class="cap cap-100"></div>
|
|
158
|
+
<div class="cap cap-100"></div>
|
|
159
|
+
<div class="cap cap-90"></div>
|
|
160
|
+
<div class="cap"></div>
|
|
161
|
+
</div>
|
|
162
|
+
</h6>
|
|
163
|
+
</div>
|
|
164
|
+
</div>
|
|
165
|
+
</li>
|
|
166
|
+
```
|
|
167
|
+
4. scrape this the same way
|
|
168
|
+
5. repeat until found "x" entries or they're too old
|
|
169
|
+
6. scrape checkins to see who's recently drunk. Could keep track of serving style (draft, taster) to only match against tap menus.
|
|
170
|
+
|
|
171
|
+
## mutiple menus all displayed
|
|
172
|
+
|
|
173
|
+
## mutiple menus selectable
|
|
174
|
+
|
|
175
|
+
?ng_menu_id=72f516dc-3ef0-4333-9113-1fa70c2143db
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# Example
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
def get_menus(html: HTML) -> List[str]:
|
|
183
|
+
"""Find menu names and links for a supplied venue response.
|
|
184
|
+
|
|
185
|
+
There are three types of venues:
|
|
186
|
+
1. official with menu selection. Eg, 99967 (Collaroy)
|
|
187
|
+
1. official with multiple menus. Eg, 107565 (Sweeneys)
|
|
188
|
+
2. official with one menu. Eg, 8931121 (Winona)
|
|
189
|
+
3. unofficial. Eg, 14705 (4 Pines)
|
|
190
|
+
|
|
191
|
+
Menu heirarchy:
|
|
192
|
+
menu header, eg, "The Arlington Tap List", "Hotel Sweeneys", "Beer List" (can be multiple with diff URL)
|
|
193
|
+
menu section, eg, "Currently Tapped", "Hand Pumps", "Main Bar", "WINONA Beer List" (can be multi)
|
|
194
|
+
beer
|
|
195
|
+
more beer (if more than 15)
|
|
196
|
+
```
|
|
197
|
+
```html
|
|
198
|
+
<div class="menu-header">
|
|
199
|
+
<select> <---|
|
|
200
|
+
<option value="e3e40b1b">BEER ON TAP</option> <---+-- if multi headers
|
|
201
|
+
</select> <---|
|
|
202
|
+
<p class="menu-total">BEER ON TAP</p> <--- current menu (below)
|
|
203
|
+
<span class="updated=time" data-time="xxx"></span> <--- menu header update time
|
|
204
|
+
</div>
|
|
205
|
+
|
|
206
|
+
<div class="menu-section" id="section_172559734">
|
|
207
|
+
<div class="menu-section-header">
|
|
208
|
+
<h4>Hand Pumps (Ground Floor) <span>(2 Beers)</span></h4> <--- menu section name
|
|
209
|
+
<ul class="menu-section-list" id="section-menu-list-172559734">
|
|
210
|
+
<li> <--- multiple
|
|
211
|
+
<div class="beer-info ">
|
|
212
|
+
<div class="beer-label">
|
|
213
|
+
<span class="new-trigger" data-update-at="Sun, 20 Dec 2020 10:06:18 +0000" style="display: none;">New</span>
|
|
214
|
+
</div>
|
|
215
|
+
<div class="beer-details">
|
|
216
|
+
<h5><a class="track-click" data-track="menu" data-href=":beer" href="/b/joshua-tetley-and-son-tetley-s-smooth-flow/986134">Tetley's Smooth Flow</a> <em>English Bitter</em></h5>
|
|
217
|
+
<h6>
|
|
218
|
+
<span>3.6% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery" href="/w/joshua-tetley-son/183905">Joshua Tetley & Son</a>• </span>
|
|
219
|
+
<div class="caps small" data-rating="2.829">...</div>
|
|
220
|
+
</h6>
|
|
221
|
+
</div>
|
|
222
|
+
</div>
|
|
223
|
+
<div class="beer-containers">
|
|
224
|
+
<p><strong>Serving:</strong>330ml Can</p>
|
|
225
|
+
</div>
|
|
226
|
+
</li>
|
|
227
|
+
</ul>
|
|
228
|
+
<a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="172717719" data-venue-id="99967" data-menu-id="48363844">Show More Beers</a>
|
|
229
|
+
</div>
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
1. Offical, menu selection
|
|
233
|
+
|
|
234
|
+
A menu list looke like:
|
|
235
|
+
```html
|
|
236
|
+
<div class="menu-header menu-select multi">
|
|
237
|
+
<select class="menu-selector">
|
|
238
|
+
<option value="e3e40b1b-ba2e-4eed-950e-ea13182cc4f0">BEER ON TAP</option>
|
|
239
|
+
<option value="72f516dc-3ef0-4333-9113-1fa70c2143db">The Arlington Fridge List</option>
|
|
240
|
+
</select>
|
|
241
|
+
<p class="menu-total">BEER ON TAP</p>
|
|
242
|
+
<span class="updated-time" data-time="Sun, 10 Jan 2021 04:08:10 +0000">Sunday, Jan 10, 3:08 PM</span>
|
|
243
|
+
</div>
|
|
244
|
+
|
|
245
|
+
The full menu for menu in the text of the menu-total <p> will be on this page.
|
|
246
|
+
Other menus need to be separately fetched, eg,
|
|
247
|
+
https://untappd.com/v/the-beach-club-collaroy/99967?ng_menu_id=72f516dc-3ef0-4333-9113-1fa70c2143db
|
|
248
|
+
|
|
249
|
+
A beer menu looks like:
|
|
250
|
+
|
|
251
|
+
<div class="menu-section" id="section_172646387">
|
|
252
|
+
<div class="menu-section-header">
|
|
253
|
+
<h4>The Arlington Fridge List <span>(96 Beers)</span></h4>
|
|
254
|
+
</div>
|
|
255
|
+
<ul class="menu-section-list" id="section-menu-list-172646387">
|
|
256
|
+
<li>
|
|
257
|
+
<div class="beer-info ">
|
|
258
|
+
<div class="beer-label">
|
|
259
|
+
<span class="new-trigger" data-update-at="Fri, 29 Jan 2021 04:21:54 +0000" style="display: none;">New</span>
|
|
260
|
+
<img src="https://untappd.akamaized.net/site/brewery_logos/brewery-3324_28f22.jpeg">
|
|
261
|
+
</div>
|
|
262
|
+
<div class="beer-details">
|
|
263
|
+
<h5><a class="track-click" data-track="menu" data-href=":beer" href="/b/3-ravens-brewery-nat-rav-2020-heathcote-yellow-muscat/3854903">Nat Rav 2020 Heathcote Yellow Muscat</a> <em>Sour - Other</em></h5>
|
|
264
|
+
<h6><span>7.5% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery" href="/w/3-ravens-brewery/3324">3 Ravens Brewery</a>• </span>
|
|
265
|
+
<div class="caps small" data-rating="3.924">
|
|
266
|
+
<div class="cap cap-100"></div>
|
|
267
|
+
<div class="cap cap-100"></div>
|
|
268
|
+
<div class="cap cap-100"></div>
|
|
269
|
+
<div class="cap cap-90"></div>
|
|
270
|
+
<div class="cap"></div>
|
|
271
|
+
</div>
|
|
272
|
+
</h6>
|
|
273
|
+
</div>
|
|
274
|
+
</div>
|
|
275
|
+
<div class="beer-containers">
|
|
276
|
+
<p>
|
|
277
|
+
<strong>Serving:</strong>
|
|
278
|
+
750ml Bottle
|
|
279
|
+
</p>
|
|
280
|
+
</div>
|
|
281
|
+
</li>
|
|
282
|
+
</ul>
|
|
283
|
+
<a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="172646387" data-venue-id="99967" data-menu-id="48343231">Show More Beers</a>
|
|
284
|
+
<p class="stream-loading"><span></span></p>
|
|
285
|
+
</div>
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
2. Offical, multiple menus
|
|
289
|
+
|
|
290
|
+
A menu list looke like:
|
|
291
|
+
```html
|
|
292
|
+
<div class="menu-header menu-select">
|
|
293
|
+
<p class="menu-total">Hotel Sweeneys</p>
|
|
294
|
+
<span class="updated-time" data-time="Sat, 30 Jan 2021 14:57:56 +0000">Sunday, Jan 31, 1:57 AM</span>
|
|
295
|
+
</div>
|
|
296
|
+
|
|
297
|
+
So, same details with menu-total not really being a _menu_ and the updated time but no selection
|
|
298
|
+
|
|
299
|
+
There are multiple of these sections. A single beer menu looks the same as the other menu options:
|
|
300
|
+
|
|
301
|
+
<div class="menu-section" id="section_172559734">
|
|
302
|
+
<div class="menu-section-header">
|
|
303
|
+
<h4>Hand Pumps (Ground Floor) <span>(2 Beers)</span></h4>
|
|
304
|
+
<p>We have 2 hand pumps in the 'main' downstairs bar and rotate through UK and Australian beers. Regular features are London Pride and/or ESB from Fullers. They are less carbonated and definitely worth a try.</p> </div>
|
|
305
|
+
<ul class="menu-section-list" id="section-menu-list-172559734">
|
|
306
|
+
<li>
|
|
307
|
+
: :
|
|
308
|
+
</li>
|
|
309
|
+
</ul>
|
|
310
|
+
</div>
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
3. Official, single menu
|
|
315
|
+
|
|
316
|
+
A menu list looke like:
|
|
317
|
+
```html
|
|
318
|
+
<div class="menu-header menu-select">
|
|
319
|
+
<p class="menu-total">Beer List</p>
|
|
320
|
+
<span class="updated-time" data-time="Sat, 30 Jan 2021 04:08:01 +0000">Saturday, Jan 30, 3:08 PM</span>
|
|
321
|
+
</div>
|
|
322
|
+
|
|
323
|
+
So, same details with menu-total and the updated time but no selection
|
|
324
|
+
|
|
325
|
+
A beer menu looks the same as in 1.
|
|
326
|
+
|
|
327
|
+
For official venues, extra menu pages (15 more beers) can be loaded via, eg,
|
|
328
|
+
https://untappd.com/venue/more_menu/99967/15?section_id=172646387", with headers={"x-requested-with": "XMLHttpRequest"}.
|
|
329
|
+
This will return a JSON-style response as it thinks we're AJAX
|
|
330
|
+
```
|
|
331
|
+
4. Unoffical
|
|
332
|
+
|
|
333
|
+
No menu-header <div>
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
# module structure
|
|
337
|
+
```mermaid
|
|
338
|
+
|
|
339
|
+
```
|
|
340
|
+
```mermaid
|
|
341
|
+
graph LR
|
|
342
|
+
uua(upd_user_activity) <--> bdb[(beers)]
|
|
343
|
+
uua <--> vdb[(venues)]
|
|
344
|
+
uua --> hs([html_session])
|
|
345
|
+
```
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Michael Wardman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: untappd-scraper
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Scrape various Untappd web pages and return as parsed dataclasses
|
|
5
|
+
Author-email: Wardy <wardy3@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2021 Michael Wardman
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Keywords: beer,scraper,untappd
|
|
29
|
+
Classifier: Development Status :: 4 - Beta
|
|
30
|
+
Classifier: Intended Audience :: Developers
|
|
31
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
32
|
+
Classifier: Operating System :: OS Independent
|
|
33
|
+
Classifier: Programming Language :: Python :: 3
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
35
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
36
|
+
Requires-Python: <4.0,>=3.13
|
|
37
|
+
Requires-Dist: brotli>=1.1.0
|
|
38
|
+
Requires-Dist: core>=0.1.4.3
|
|
39
|
+
Requires-Dist: haversine>=2.9.0
|
|
40
|
+
Requires-Dist: lxml[html-clean]>=5.3.2
|
|
41
|
+
Requires-Dist: parse>=1.20.2
|
|
42
|
+
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
43
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
44
|
+
Requires-Dist: ratelim>=0.1.6
|
|
45
|
+
Requires-Dist: requests-cache>=1.2.1
|
|
46
|
+
Requires-Dist: requests-html>=0.10.0
|
|
47
|
+
Requires-Dist: tenacity>=9.1.2
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# Untappd Scraper
|
|
51
|
+
|
|
52
|
+
Web scrape public Untappd pages into data classes.
|
|
53
|
+
|
|
54
|
+
## Quickstart
|
|
55
|
+
|
|
56
|
+
### User queries
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from untappd_scraper.user import User
|
|
60
|
+
|
|
61
|
+
users = ["gregavola"]
|
|
62
|
+
|
|
63
|
+
for user_id in users:
|
|
64
|
+
user = User(user_id)
|
|
65
|
+
print(f"\n{user.name=}")
|
|
66
|
+
|
|
67
|
+
for checkin in user.activity():
|
|
68
|
+
print(f"\n\t{checkin.name=}\n\t\t{checkin}")
|
|
69
|
+
|
|
70
|
+
print("\n\nLists:")
|
|
71
|
+
|
|
72
|
+
for userlist in user.lists():
|
|
73
|
+
print(f"\n\t{userlist.name=} {userlist.num_items=}")
|
|
74
|
+
|
|
75
|
+
print("\n\nRecent Venues:")
|
|
76
|
+
|
|
77
|
+
for venue in user.venue_history():
|
|
78
|
+
ago = datetime.now().date() - venue.last_visit
|
|
79
|
+
print(
|
|
80
|
+
f"\n\t{venue.name=} {venue.last_visit.strftime('%b %d %Y')} "
|
|
81
|
+
+ f"({ago.days} days ago)"
|
|
82
|
+
)```
|
|
83
|
+
|
|
84
|
+
print("\n\nRecent Uniques:")
|
|
85
|
+
|
|
86
|
+
for beer in user.beer_history():
|
|
87
|
+
print(f"\t{beer}, {beer.total_checkins=}")
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Venue queries
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from untappd_scraper.venue import Venue
|
|
94
|
+
|
|
95
|
+
venue_ids = [
|
|
96
|
+
14705, # 4 pines
|
|
97
|
+
99967, # collaroy
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
for venue_id in venue_ids:
|
|
101
|
+
venue = Venue(venue_id)
|
|
102
|
+
print(f"\n{venue.name=}")
|
|
103
|
+
|
|
104
|
+
for num, checkin in enumerate(venue.activity(), start=1):
|
|
105
|
+
print(f"\n\t{num=}\t{checkin.name=}\n\t\t{checkin}")
|
|
106
|
+
|
|
107
|
+
# Load a venue by name. Try to be as unambiguous as possible
|
|
108
|
+
venue = Venue.from_name("hotel sweeney")
|
|
109
|
+
print(f"\n{venue.name=} {venue.verified=}\t{venue.categories=}\n")
|
|
110
|
+
print("\n".join([str(beer) for beer in venue.activity()]))
|
|
111
|
+
|
|
112
|
+
for menu in venue.menus():
|
|
113
|
+
print(f"\n{menu.selection=} / {menu.name=}")
|
|
114
|
+
|
|
115
|
+
print("\n\t\t", end="")
|
|
116
|
+
print("\n\t\t".join(str(beer) for beer in menu.beers))
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Notes
|
|
120
|
+
|
|
121
|
+
untappd-scraper is just that - a scraper. It doesn't store data. So, for example,
|
|
122
|
+
if you query a user's unique beer history (with `User.beer_history()`) you will not
|
|
123
|
+
be able to see the entire history. Just what is public on the web, which is the most
|
|
124
|
+
recent 25 uniques.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Untappd Scraper
|
|
2
|
+
|
|
3
|
+
Web scrape public Untappd pages into data classes.
|
|
4
|
+
|
|
5
|
+
## Quickstart
|
|
6
|
+
|
|
7
|
+
### User queries
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from untappd_scraper.user import User
|
|
11
|
+
|
|
12
|
+
users = ["gregavola"]
|
|
13
|
+
|
|
14
|
+
for user_id in users:
|
|
15
|
+
user = User(user_id)
|
|
16
|
+
print(f"\n{user.name=}")
|
|
17
|
+
|
|
18
|
+
for checkin in user.activity():
|
|
19
|
+
print(f"\n\t{checkin.name=}\n\t\t{checkin}")
|
|
20
|
+
|
|
21
|
+
print("\n\nLists:")
|
|
22
|
+
|
|
23
|
+
for userlist in user.lists():
|
|
24
|
+
print(f"\n\t{userlist.name=} {userlist.num_items=}")
|
|
25
|
+
|
|
26
|
+
print("\n\nRecent Venues:")
|
|
27
|
+
|
|
28
|
+
for venue in user.venue_history():
|
|
29
|
+
ago = datetime.now().date() - venue.last_visit
|
|
30
|
+
print(
|
|
31
|
+
f"\n\t{venue.name=} {venue.last_visit.strftime('%b %d %Y')} "
|
|
32
|
+
+ f"({ago.days} days ago)"
|
|
33
|
+
)```
|
|
34
|
+
|
|
35
|
+
print("\n\nRecent Uniques:")
|
|
36
|
+
|
|
37
|
+
for beer in user.beer_history():
|
|
38
|
+
print(f"\t{beer}, {beer.total_checkins=}")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### Venue queries
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from untappd_scraper.venue import Venue
|
|
45
|
+
|
|
46
|
+
venue_ids = [
|
|
47
|
+
14705, # 4 pines
|
|
48
|
+
99967, # collaroy
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
for venue_id in venue_ids:
|
|
52
|
+
venue = Venue(venue_id)
|
|
53
|
+
print(f"\n{venue.name=}")
|
|
54
|
+
|
|
55
|
+
for num, checkin in enumerate(venue.activity(), start=1):
|
|
56
|
+
print(f"\n\t{num=}\t{checkin.name=}\n\t\t{checkin}")
|
|
57
|
+
|
|
58
|
+
# Load a venue by name. Try to be as unambiguous as possible
|
|
59
|
+
venue = Venue.from_name("hotel sweeney")
|
|
60
|
+
print(f"\n{venue.name=} {venue.verified=}\t{venue.categories=}\n")
|
|
61
|
+
print("\n".join([str(beer) for beer in venue.activity()]))
|
|
62
|
+
|
|
63
|
+
for menu in venue.menus():
|
|
64
|
+
print(f"\n{menu.selection=} / {menu.name=}")
|
|
65
|
+
|
|
66
|
+
print("\n\t\t", end="")
|
|
67
|
+
print("\n\t\t".join(str(beer) for beer in menu.beers))
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Notes
|
|
71
|
+
|
|
72
|
+
untappd-scraper is just that - a scraper. It doesn't store data. So, for example,
|
|
73
|
+
if you query a user's unique beer history (with `User.beer_history()`) you will not
|
|
74
|
+
be able to see the entire history. Just what is public on the web, which is the most
|
|
75
|
+
recent 25 uniques.
|