untappd-scraper 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. untappd_scraper-0.4.0/.gitignore +31 -0
  2. untappd_scraper-0.4.0/.gitlab-ci.yml +47 -0
  3. untappd_scraper-0.4.0/DESIGN.md +345 -0
  4. untappd_scraper-0.4.0/Dockerfile +6 -0
  5. untappd_scraper-0.4.0/LICENSE +21 -0
  6. untappd_scraper-0.4.0/PKG-INFO +124 -0
  7. untappd_scraper-0.4.0/README.md +75 -0
  8. untappd_scraper-0.4.0/pyproject.toml +79 -0
  9. untappd_scraper-0.4.0/query_user.py +16 -0
  10. untappd_scraper-0.4.0/ruff.toml +50 -0
  11. untappd_scraper-0.4.0/tests/__init__.py +0 -0
  12. untappd_scraper-0.4.0/tests/conftest.py +239 -0
  13. untappd_scraper-0.4.0/tests/html/beer.html +2499 -0
  14. untappd_scraper-0.4.0/tests/html/fridge-list.html +1821 -0
  15. untappd_scraper-0.4.0/tests/html/user-beer-history.html +1816 -0
  16. untappd_scraper-0.4.0/tests/html/user-list-1.html +1821 -0
  17. untappd_scraper-0.4.0/tests/html/user-list-2.html +1816 -0
  18. untappd_scraper-0.4.0/tests/html/user-venue-history.html +1960 -0
  19. untappd_scraper-0.4.0/tests/html/user.html +1323 -0
  20. untappd_scraper-0.4.0/tests/html/userlists.html +852 -0
  21. untappd_scraper-0.4.0/tests/html/venue_unv.html +2341 -0
  22. untappd_scraper-0.4.0/tests/html/venue_ver.html +3246 -0
  23. untappd_scraper-0.4.0/tests/html/venue_ver_nest.html +2004 -0
  24. untappd_scraper-0.4.0/tests/html/wishlist.html +1358 -0
  25. untappd_scraper-0.4.0/tests/test_beer.py +36 -0
  26. untappd_scraper-0.4.0/tests/test_user.py +99 -0
  27. untappd_scraper-0.4.0/tests/test_user_beer_history.py +19 -0
  28. untappd_scraper-0.4.0/tests/test_user_lists.py +70 -0
  29. untappd_scraper-0.4.0/tests/test_user_utils.py +9 -0
  30. untappd_scraper-0.4.0/tests/test_user_venue_history.py +20 -0
  31. untappd_scraper-0.4.0/tests/test_venue.py +89 -0
  32. untappd_scraper-0.4.0/try_beer_details.py +10 -0
  33. untappd_scraper-0.4.0/try_user.py +39 -0
  34. untappd_scraper-0.4.0/try_user_activity.py +21 -0
  35. untappd_scraper-0.4.0/try_user_details.py +20 -0
  36. untappd_scraper-0.4.0/try_user_had.py +21 -0
  37. untappd_scraper-0.4.0/try_user_lists.py +31 -0
  38. untappd_scraper-0.4.0/try_user_recent_venues.py +21 -0
  39. untappd_scraper-0.4.0/try_venue.py +115 -0
  40. untappd_scraper-0.4.0/try_venue_activity.py +15 -0
  41. untappd_scraper-0.4.0/try_venue_details.py +12 -0
  42. untappd_scraper-0.4.0/try_venue_menus.py +46 -0
  43. untappd_scraper-0.4.0/untappd_scraper/__init__.py +1 -0
  44. untappd_scraper-0.4.0/untappd_scraper/beer.py +193 -0
  45. untappd_scraper-0.4.0/untappd_scraper/html_session.py +231 -0
  46. untappd_scraper-0.4.0/untappd_scraper/logging_config.py +78 -0
  47. untappd_scraper-0.4.0/untappd_scraper/logs.yml +59 -0
  48. untappd_scraper-0.4.0/untappd_scraper/py.typed +0 -0
  49. untappd_scraper-0.4.0/untappd_scraper/structs/__init__.py +0 -0
  50. untappd_scraper-0.4.0/untappd_scraper/structs/mixins.py +63 -0
  51. untappd_scraper-0.4.0/untappd_scraper/structs/other.py +27 -0
  52. untappd_scraper-0.4.0/untappd_scraper/structs/web.py +183 -0
  53. untappd_scraper-0.4.0/untappd_scraper/user.py +173 -0
  54. untappd_scraper-0.4.0/untappd_scraper/user_beer_history.py +112 -0
  55. untappd_scraper-0.4.0/untappd_scraper/user_lists.py +268 -0
  56. untappd_scraper-0.4.0/untappd_scraper/user_lists_details.py +117 -0
  57. untappd_scraper-0.4.0/untappd_scraper/user_utils.py +21 -0
  58. untappd_scraper-0.4.0/untappd_scraper/user_venue_history.py +67 -0
  59. untappd_scraper-0.4.0/untappd_scraper/venue.py +168 -0
  60. untappd_scraper-0.4.0/untappd_scraper/venue_menus.py +321 -0
  61. untappd_scraper-0.4.0/untappd_scraper/venue_utils.py +14 -0
  62. untappd_scraper-0.4.0/untappd_scraper/web.py +113 -0
  63. untappd_scraper-0.4.0/uv.lock +788 -0
@@ -0,0 +1,31 @@
1
+ # temp / uninteresting files
2
+ .python-version
3
+ .vscode/
4
+ .vstags
5
+ __pycache__
6
+ .mypy_cache
7
+
8
+ # distribution
9
+ dist/
10
+ pip/
11
+ pip.zip
12
+
13
+ # Sphinx
14
+ docs/_build
15
+ docs/_static
16
+ docs/_templates
17
+
18
+ # project
19
+ html.sqlite
20
+ *.log
21
+ a.html
22
+ b.html
23
+ conf2.py
24
+ fadel.json
25
+ u.json
26
+
27
+ .coverage
28
+ coverage.xml
29
+ .testmondata
30
+ cov.xml
31
+ .DS_Store
@@ -0,0 +1,47 @@
1
+ stages:
2
+ - test
3
+
4
+ variables:
5
+ PYTHONDONTWRITEBYTECODE: 1
6
+ PYTHONPATH: "./:./UntappdUniques:./tests"
7
+ TZ: Australia/Sydney
8
+
9
+ # UV
10
+ UV_VERSION: 0.6.3
11
+ PYTHON_VERSION: 3.13
12
+ BASE_LAYER: bookworm-slim
13
+ # GitLab CI creates a separate mountpoint for the build directory,
14
+ # so we need to copy instead of using hard links.
15
+ UV_LINK_MODE: copy
16
+
17
+ before_script:
18
+ - uname -a
19
+ - python --version
20
+ - uv --version
21
+ - set -u
22
+ - export UV_INDEX="https://__token__:${GITLAB_PYPI_PAT}@gitlab.com/api/v4/projects/23576087/packages/pypi/simple"
23
+
24
+ test-with-uv:
25
+ image: ghcr.io/astral-sh/uv:$UV_VERSION-python$PYTHON_VERSION-$BASE_LAYER
26
+ variables:
27
+ UV_CACHE_DIR: .uv-cache
28
+ cache:
29
+ - key:
30
+ files:
31
+ - uv.lock
32
+ - .venv
33
+ paths:
34
+ - $UV_CACHE_DIR
35
+
36
+ stage: test
37
+ script:
38
+ # - env | sort
39
+ # Your `uv` commands
40
+ - uv sync --group test
41
+ # - uv pip list
42
+ - uv run coverage run -m pytest # Run all tests
43
+ - uv run coverage xml
44
+ - uv run coverage report
45
+ # recommended to reduce cache size
46
+ - uv cache prune --ci
47
+ coverage: '/TOTAL.*\s+(\d+%)/'
@@ -0,0 +1,345 @@
1
+ # What is Scraped?
2
+ | type | url | web struct | note |
3
+ | ---------------- | ------------------------------ | ---------------------------------- | ---------------------------------------------- |
4
+ | beer details | beer/{beer_id} | WebBeerDetails |
5
+ | user details | user/{user_id} | WebUserDetails<br> WebActivityBeer | has activity too |
6
+ | user had beers | user/{user_id}/beers | WebUserHadBeer |
7
+ | user list names | user/{user_id}/lists | WebUserList |
8
+ | user list beers | user/{user_id}/lists/{list_id} | WebUserListBeer |
9
+ | user venues | user/{user_id}/venues | WebUserVenueHistory |
10
+ | venue activity | venue/{venue_id}/activity | WebActivityBeer | unverified venue on same page as venue details |
11
+ | venue details | venue/{venue_id} | WebVenueDetails |
12
+ | venue menu names | venue/{venue_id} | WebVenueMenu |
13
+ | venue menu beers | venue/{venue_id} | WebVenueMenuBeer | may be a different URL |
14
+
15
+ # Loop
16
+ ```
17
+ # load menu beers
18
+ if has menu-header
19
+ if has menu-selector
20
+ for each menu-selector
21
+ load menu
22
+ for each menu-section
23
+ process menu
24
+ else
25
+ for each menu-section
26
+ process menu
27
+
28
+ # process menu (sub)
29
+ for each menu-section-header (collaroy packed beers)
30
+ process menu-section-header
31
+ for each menu-section-list
32
+ while more beer
33
+ for each beer in list
34
+ process beer
35
+ if show-more-section and still_want_more():
36
+ load next page
37
+
38
+ # load checkins
39
+ if has menu
40
+ load activity page
41
+
42
+ for each checkin
43
+ log beer, time, serving, rating
44
+ ```
45
+ # HTML
46
+ ## Menus
47
+
48
+ ### examples
49
+ | venue | type |
50
+ | -------- | ---------------- |
51
+ | 4 pines | unofficial |
52
+ | winona | single |
53
+ | sweeneys | multi |
54
+ | collaroy | multi selectable |
55
+
56
+ ### menu tags
57
+ ```
58
+ menu-header
59
+
60
+ missing for 4 pines
61
+ -header menu-select for winona
62
+ menu-select for sweeneys
63
+ menu-select multi for collaroy
64
+
65
+ menu-selector for collaroy
66
+ <select class=‘menu-selector’>
67
+ <option value=“xxx”>menu name</option>
68
+
69
+ menu-section-header
70
+ <h4>menu name<span>xx</span></h4>
71
+
72
+ missing for 4 pines
73
+ single for winona
74
+ multiple for sweeneys
75
+ single for collaroy
76
+
77
+ menu-section-list
78
+ <ul><li><div class=“beer-info”>
79
+ <div class=“beer-label”><span data-update-at=“date”></span>
80
+ <div class=“beer-details”>
81
+ <h5><a href=“xx”>beer name</a><em>style</em></h5>
82
+ <h6><span><a href=“brewery”>brewery</a><div class=“caps small” data-rating=“3.949”></h6>
83
+ </ul>
84
+ <a class=“more show-more-section” data-section-id=“123” data-venue-id=“123” data-menu-id=“123”>
85
+ ```
86
+ ### activity tags
87
+ ```html
88
+ <div class=“checkin”>
89
+ <div class=“top”>
90
+ <p class=“text”><a class=“user”>Bob</a> is drinking a/n <a>beer</a> by <a>brewery</a> at <a>venue</a></p>
91
+ <div class=“checkin-comment><div class="rating-serving">
92
+ <p class="serving"><span>Can</span></p>
93
+ <div class="caps " data-rating="4.25">
94
+ </div>
95
+ <div class="feedback">
96
+ <div class="bottom">
97
+ <a href="/user/Martinlife/checkin/991966052" class="time timezoner track-click" data-gregtime="Sun, 31 Jan 2021 05:35:45 +0000">31 Jan 21</a>
98
+ ```
99
+ ## 4 Pines (no menu)
100
+
101
+ # Menu systems
102
+
103
+ There are several different presentations of a venue's menu:
104
+ | type | example | id | url |
105
+ | ------------------------- | ------------------- | ------- | --------------------------------------------------- |
106
+ | unverified | 4 Pines brewpub | 14705 | https://untappd.com/v/4-pines-brewpub/14705 |
107
+ | single menu | Winona | 8931121 | https://untappd.com/v/winona/8931121 |
108
+ | multi menu, all displayed | Sweeneys | 107565 | https://untappd.com/v/hotel-sweeney-s/107565 |
109
+ | multi menu, selectable | Collaroy Beach Club | 99967 | https://untappd.com/v/the-beach-club-collaroy/99967 |
110
+
111
+ ## unverified
112
+ Not much to do here.
113
+ 1. scrape the checkins. Can get user's personal rating (but use global if available), and check in time easily
114
+ 2. take note of "recent". < 1 week?
115
+ 3. can't page, so just take the 20
116
+ ## single menu
117
+ Some of these are huge. Winona has 554 on their menu but they aren't necessarily available - they just don't clean it up. So only scrape 100(?) or only since a certain date (available in HTML)
118
+ 1. get beer list name
119
+ 2. get beers on first page, capturing
120
+ - global rating
121
+ - name
122
+ - brewery
123
+ - style
124
+ - date updated in
125
+ ```html
126
+ <span class="new-trigger" data-update-at="Mon, 30 Nov 2020 05:44:33 +0000" style="display: none;">New</span>
127
+ ```
128
+ 3. repeat by "pressing" the "Show More Beers". This actually is a request for XHR data
129
+ ```bash
130
+ curl -XGET -H "x-requested-with: XMLHttpRequest" "https://untappd.com/venue/more_menu/8931121/15?section_id=178512830"
131
+ ```
132
+ Subsequent pressing gets a url like `https://untappd.com/venue/more_menu/8931121/30?section_id=178512830`. '8931121' is the venue ID and stays the same. Note the 15 changed to a 30, which is the start number (then goes 45, 60, ...). Section ID 178512830 stays the same too and needs to be scraped from the "more" button (or the other tags nearby).
133
+ ```html
134
+ <a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="178512830" data-venue-id="8931121" data-menu-id="50008188" style="display: block;">Show More Beers</a>
135
+ ```
136
+ This returns a JSON string with HTML in it, eg,
137
+ ```json
138
+ {"count":15,"view":"\t\t<li>\n\t\t\t<div class=\"beer-info \">\n\t\t\t <div class=...}
139
+ ```
140
+ HTML is a bunch of `<li>` tags, eg,
141
+ ```html
142
+ <li>
143
+ <div class="beer-info ">
144
+ <div class="beer-label">
145
+ <span class="new-trigger" data-update-at="Thu, 25 Feb 2021 08:52:44 +0000" style="display: none;">New</span>
146
+ </div>
147
+ <div class="beer-details">
148
+ <h5><a class="track-click" data-track="menu" data-href=":beer"
149
+ href="/b/garage-project-hazy-daze-amarillo-and-mosaic/4185296">Hazy Daze- Amarillo & Mosaic</a>
150
+ <em>Pale Ale - New England</em></h5>
151
+ <h6><span>5.8% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery"
152
+ href="/w/garage-project/14539">Garage Project</a>
153
+ • </span>
154
+
155
+ <div class="caps small" data-rating="3.946">
156
+ <div class="cap cap-100"></div>
157
+ <div class="cap cap-100"></div>
158
+ <div class="cap cap-100"></div>
159
+ <div class="cap cap-90"></div>
160
+ <div class="cap"></div>
161
+ </div>
162
+ </h6>
163
+ </div>
164
+ </div>
165
+ </li>
166
+ ```
167
+ 4. scrape this the same way
168
+ 5. repeat until found "x" entries or they're too old
169
+ 6. scrape checkins to see who's recently drunk. Could keep track of serving style (draft, taster) to only match against tap menus.
170
+
171
+ ## mutiple menus all displayed
172
+
173
+ ## mutiple menus selectable
174
+
175
+ ?ng_menu_id=72f516dc-3ef0-4333-9113-1fa70c2143db
176
+
177
+
178
+
179
+ # Example
180
+
181
+ ```python
182
+ def get_menus(html: HTML) -> List[str]:
183
+ """Find menu names and links for a supplied venue response.
184
+
185
+ There are three types of venues:
186
+ 1. official with menu selection. Eg, 99967 (Collaroy)
187
+ 1. official with multiple menus. Eg, 107565 (Sweeneys)
188
+ 2. official with one menu. Eg, 8931121 (Winona)
189
+ 3. unofficial. Eg, 14705 (4 Pines)
190
+
191
+ Menu heirarchy:
192
+ menu header, eg, "The Arlington Tap List", "Hotel Sweeneys", "Beer List" (can be multiple with diff URL)
193
+ menu section, eg, "Currently Tapped", "Hand Pumps", "Main Bar", "WINONA Beer List" (can be multi)
194
+ beer
195
+ more beer (if more than 15)
196
+ ```
197
+ ```html
198
+ <div class="menu-header">
199
+ <select> <---|
200
+ <option value="e3e40b1b">BEER ON TAP</option> <---+-- if multi headers
201
+ </select> <---|
202
+ <p class="menu-total">BEER ON TAP</p> <--- current menu (below)
203
+ <span class="updated=time" data-time="xxx"></span> <--- menu header update time
204
+ </div>
205
+
206
+ <div class="menu-section" id="section_172559734">
207
+ <div class="menu-section-header">
208
+ <h4>Hand Pumps (Ground Floor) <span>(2 Beers)</span></h4> <--- menu section name
209
+ <ul class="menu-section-list" id="section-menu-list-172559734">
210
+ <li> <--- multiple
211
+ <div class="beer-info ">
212
+ <div class="beer-label">
213
+ <span class="new-trigger" data-update-at="Sun, 20 Dec 2020 10:06:18 +0000" style="display: none;">New</span>
214
+ </div>
215
+ <div class="beer-details">
216
+ <h5><a class="track-click" data-track="menu" data-href=":beer" href="/b/joshua-tetley-and-son-tetley-s-smooth-flow/986134">Tetley's Smooth Flow</a> <em>English Bitter</em></h5>
217
+ <h6>
218
+ <span>3.6% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery" href="/w/joshua-tetley-son/183905">Joshua Tetley &amp; Son</a>• </span>
219
+ <div class="caps small" data-rating="2.829">...</div>
220
+ </h6>
221
+ </div>
222
+ </div>
223
+ <div class="beer-containers">
224
+ <p><strong>Serving:</strong>330ml Can</p>
225
+ </div>
226
+ </li>
227
+ </ul>
228
+ <a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="172717719" data-venue-id="99967" data-menu-id="48363844">Show More Beers</a>
229
+ </div>
230
+ ```
231
+
232
+ 1. Offical, menu selection
233
+
234
+ A menu list looke like:
235
+ ```html
236
+ <div class="menu-header menu-select multi">
237
+ <select class="menu-selector">
238
+ <option value="e3e40b1b-ba2e-4eed-950e-ea13182cc4f0">BEER ON TAP</option>
239
+ <option value="72f516dc-3ef0-4333-9113-1fa70c2143db">The Arlington Fridge List</option>
240
+ </select>
241
+ <p class="menu-total">BEER ON TAP</p>
242
+ <span class="updated-time" data-time="Sun, 10 Jan 2021 04:08:10 +0000">Sunday, Jan 10, 3:08 PM</span>
243
+ </div>
244
+
245
+ The full menu for menu in the text of the menu-total <p> will be on this page.
246
+ Other menus need to be separately fetched, eg,
247
+ https://untappd.com/v/the-beach-club-collaroy/99967?ng_menu_id=72f516dc-3ef0-4333-9113-1fa70c2143db
248
+
249
+ A beer menu looks like:
250
+
251
+ <div class="menu-section" id="section_172646387">
252
+ <div class="menu-section-header">
253
+ <h4>The Arlington Fridge List <span>(96 Beers)</span></h4>
254
+ </div>
255
+ <ul class="menu-section-list" id="section-menu-list-172646387">
256
+ <li>
257
+ <div class="beer-info ">
258
+ <div class="beer-label">
259
+ <span class="new-trigger" data-update-at="Fri, 29 Jan 2021 04:21:54 +0000" style="display: none;">New</span>
260
+ <img src="https://untappd.akamaized.net/site/brewery_logos/brewery-3324_28f22.jpeg">
261
+ </div>
262
+ <div class="beer-details">
263
+ <h5><a class="track-click" data-track="menu" data-href=":beer" href="/b/3-ravens-brewery-nat-rav-2020-heathcote-yellow-muscat/3854903">Nat Rav 2020 Heathcote Yellow Muscat</a> <em>Sour - Other</em></h5>
264
+ <h6><span>7.5% ABV • N/A IBU • <a class="track-click" data-track="menu" data-href=":brewery" href="/w/3-ravens-brewery/3324">3 Ravens Brewery</a>• </span>
265
+ <div class="caps small" data-rating="3.924">
266
+ <div class="cap cap-100"></div>
267
+ <div class="cap cap-100"></div>
268
+ <div class="cap cap-100"></div>
269
+ <div class="cap cap-90"></div>
270
+ <div class="cap"></div>
271
+ </div>
272
+ </h6>
273
+ </div>
274
+ </div>
275
+ <div class="beer-containers">
276
+ <p>
277
+ <strong>Serving:</strong>
278
+ 750ml Bottle
279
+ </p>
280
+ </div>
281
+ </li>
282
+ </ul>
283
+ <a href="javascript:void(0);" class="yellow button more show-more-section track-click" data-track="venue" data-href=":moremenu" data-section-id="172646387" data-venue-id="99967" data-menu-id="48343231">Show More Beers</a>
284
+ <p class="stream-loading"><span></span></p>
285
+ </div>
286
+ ```
287
+
288
+ 2. Offical, multiple menus
289
+
290
+ A menu list looke like:
291
+ ```html
292
+ <div class="menu-header menu-select">
293
+ <p class="menu-total">Hotel Sweeneys</p>
294
+ <span class="updated-time" data-time="Sat, 30 Jan 2021 14:57:56 +0000">Sunday, Jan 31, 1:57 AM</span>
295
+ </div>
296
+
297
+ So, same details with menu-total not really being a _menu_ and the updated time but no selection
298
+
299
+ There are multiple of these sections. A single beer menu looks the same as the other menu options:
300
+
301
+ <div class="menu-section" id="section_172559734">
302
+ <div class="menu-section-header">
303
+ <h4>Hand Pumps (Ground Floor) <span>(2 Beers)</span></h4>
304
+ <p>We have 2 hand pumps in the 'main' downstairs bar and rotate through UK and Australian beers. Regular features are London Pride and/or ESB from Fullers. They are less carbonated and definitely worth a try.</p> </div>
305
+ <ul class="menu-section-list" id="section-menu-list-172559734">
306
+ <li>
307
+ : :
308
+ </li>
309
+ </ul>
310
+ </div>
311
+ ```
312
+
313
+
314
+ 3. Official, single menu
315
+
316
+ A menu list looke like:
317
+ ```html
318
+ <div class="menu-header menu-select">
319
+ <p class="menu-total">Beer List</p>
320
+ <span class="updated-time" data-time="Sat, 30 Jan 2021 04:08:01 +0000">Saturday, Jan 30, 3:08 PM</span>
321
+ </div>
322
+
323
+ So, same details with menu-total and the updated time but no selection
324
+
325
+ A beer menu looks the same as in 1.
326
+
327
+ For official venues, extra menu pages (15 more beers) can be loaded via, eg,
328
+ https://untappd.com/venue/more_menu/99967/15?section_id=172646387", with headers={"x-requested-with": "XMLHttpRequest"}.
329
+ This will return a JSON-style response as it thinks we're AJAX
330
+ ```
331
+ 4. Unoffical
332
+
333
+ No menu-header <div>
334
+
335
+
336
+ # module structure
337
+ ```mermaid
338
+
339
+ ```
340
+ ```mermaid
341
+ graph LR
342
+ uua(upd_user_activity) <--> bdb[(beers)]
343
+ uua <--> vdb[(venues)]
344
+ uua --> hs([html_session])
345
+ ```
@@ -0,0 +1,6 @@
1
+ FROM warbin/poetry:390
2
+
3
+ COPY pyproject.toml ./
4
+ COPY poetry.lock ./
5
+
6
+ RUN poetry install --only main,test
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2021 Michael Wardman
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.4
2
+ Name: untappd-scraper
3
+ Version: 0.4.0
4
+ Summary: Scrape various Untappd web pages and return as parsed dataclasses
5
+ Author-email: Wardy <wardy3@gmail.com>
6
+ License: MIT License
7
+
8
+ Copyright (c) 2021 Michael Wardman
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+ License-File: LICENSE
28
+ Keywords: beer,scraper,untappd
29
+ Classifier: Development Status :: 4 - Beta
30
+ Classifier: Intended Audience :: Developers
31
+ Classifier: License :: OSI Approved :: MIT License
32
+ Classifier: Operating System :: OS Independent
33
+ Classifier: Programming Language :: Python :: 3
34
+ Classifier: Programming Language :: Python :: 3.13
35
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
36
+ Requires-Python: <4.0,>=3.13
37
+ Requires-Dist: brotli>=1.1.0
38
+ Requires-Dist: core>=0.1.4.3
39
+ Requires-Dist: haversine>=2.9.0
40
+ Requires-Dist: lxml[html-clean]>=5.3.2
41
+ Requires-Dist: parse>=1.20.2
42
+ Requires-Dist: python-dateutil>=2.9.0.post0
43
+ Requires-Dist: pyyaml>=6.0.2
44
+ Requires-Dist: ratelim>=0.1.6
45
+ Requires-Dist: requests-cache>=1.2.1
46
+ Requires-Dist: requests-html>=0.10.0
47
+ Requires-Dist: tenacity>=9.1.2
48
+ Description-Content-Type: text/markdown
49
+
50
+ # Untappd Scraper
51
+
52
+ Web scrape public Untappd pages into data classes.
53
+
54
+ ## Quickstart
55
+
56
+ ### User queries
57
+
58
+ ```python
59
+ from untappd_scraper.user import User
60
+
61
+ users = ["gregavola"]
62
+
63
+ for user_id in users:
64
+ user = User(user_id)
65
+ print(f"\n{user.name=}")
66
+
67
+ for checkin in user.activity():
68
+ print(f"\n\t{checkin.name=}\n\t\t{checkin}")
69
+
70
+ print("\n\nLists:")
71
+
72
+ for userlist in user.lists():
73
+ print(f"\n\t{userlist.name=} {userlist.num_items=}")
74
+
75
+ print("\n\nRecent Venues:")
76
+
77
+ for venue in user.venue_history():
78
+ ago = datetime.now().date() - venue.last_visit
79
+ print(
80
+ f"\n\t{venue.name=} {venue.last_visit.strftime('%b %d %Y')} "
81
+ + f"({ago.days} days ago)"
82
+ )```
83
+
84
+ print("\n\nRecent Uniques:")
85
+
86
+ for beer in user.beer_history():
87
+ print(f"\t{beer}, {beer.total_checkins=}")
88
+ ```
89
+
90
+ ### Venue queries
91
+
92
+ ```python
93
+ from untappd_scraper.venue import Venue
94
+
95
+ venue_ids = [
96
+ 14705, # 4 pines
97
+ 99967, # collaroy
98
+ ]
99
+
100
+ for venue_id in venue_ids:
101
+ venue = Venue(venue_id)
102
+ print(f"\n{venue.name=}")
103
+
104
+ for num, checkin in enumerate(venue.activity(), start=1):
105
+ print(f"\n\t{num=}\t{checkin.name=}\n\t\t{checkin}")
106
+
107
+ # Load a venue by name. Try to be as unambiguous as possible
108
+ venue = Venue.from_name("hotel sweeney")
109
+ print(f"\n{venue.name=} {venue.verified=}\t{venue.categories=}\n")
110
+ print("\n".join([str(beer) for beer in venue.activity()]))
111
+
112
+ for menu in venue.menus():
113
+ print(f"\n{menu.selection=} / {menu.name=}")
114
+
115
+ print("\n\t\t", end="")
116
+ print("\n\t\t".join(str(beer) for beer in menu.beers))
117
+ ```
118
+
119
+ ## Notes
120
+
121
+ untappd-scraper is just that - a scraper. It doesn't store data. So, for example,
122
+ if you query a user's unique beer history (with `User.beer_history()`) you will not
123
+ be able to see the entire history. Just what is public on the web, which is the most
124
+ recent 25 uniques.
@@ -0,0 +1,75 @@
1
+ # Untappd Scraper
2
+
3
+ Web scrape public Untappd pages into data classes.
4
+
5
+ ## Quickstart
6
+
7
+ ### User queries
8
+
9
+ ```python
10
+ from untappd_scraper.user import User
11
+
12
+ users = ["gregavola"]
13
+
14
+ for user_id in users:
15
+ user = User(user_id)
16
+ print(f"\n{user.name=}")
17
+
18
+ for checkin in user.activity():
19
+ print(f"\n\t{checkin.name=}\n\t\t{checkin}")
20
+
21
+ print("\n\nLists:")
22
+
23
+ for userlist in user.lists():
24
+ print(f"\n\t{userlist.name=} {userlist.num_items=}")
25
+
26
+ print("\n\nRecent Venues:")
27
+
28
+ for venue in user.venue_history():
29
+ ago = datetime.now().date() - venue.last_visit
30
+ print(
31
+ f"\n\t{venue.name=} {venue.last_visit.strftime('%b %d %Y')} "
32
+ + f"({ago.days} days ago)"
33
+ )```
34
+
35
+ print("\n\nRecent Uniques:")
36
+
37
+ for beer in user.beer_history():
38
+ print(f"\t{beer}, {beer.total_checkins=}")
39
+ ```
40
+
41
+ ### Venue queries
42
+
43
+ ```python
44
+ from untappd_scraper.venue import Venue
45
+
46
+ venue_ids = [
47
+ 14705, # 4 pines
48
+ 99967, # collaroy
49
+ ]
50
+
51
+ for venue_id in venue_ids:
52
+ venue = Venue(venue_id)
53
+ print(f"\n{venue.name=}")
54
+
55
+ for num, checkin in enumerate(venue.activity(), start=1):
56
+ print(f"\n\t{num=}\t{checkin.name=}\n\t\t{checkin}")
57
+
58
+ # Load a venue by name. Try to be as unambiguous as possible
59
+ venue = Venue.from_name("hotel sweeney")
60
+ print(f"\n{venue.name=} {venue.verified=}\t{venue.categories=}\n")
61
+ print("\n".join([str(beer) for beer in venue.activity()]))
62
+
63
+ for menu in venue.menus():
64
+ print(f"\n{menu.selection=} / {menu.name=}")
65
+
66
+ print("\n\t\t", end="")
67
+ print("\n\t\t".join(str(beer) for beer in menu.beers))
68
+ ```
69
+
70
+ ## Notes
71
+
72
+ untappd-scraper is just that - a scraper. It doesn't store data. So, for example,
73
+ if you query a user's unique beer history (with `User.beer_history()`) you will not
74
+ be able to see the entire history. Just what is public on the web, which is the most
75
+ recent 25 uniques.