untappd-scraper 0.4.0__py3-none-any.whl

untappd_scraper/__init__.py

@@ -0,0 +1 @@
+"""Untappd Scraper functions."""

untappd_scraper/beer.py

@@ -0,0 +1,193 @@
+"""Untappd beers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from core.mixins import SimpleRepr
+from dateutil.parser import parse as parse_date
+
+from untappd_scraper.html_session import get
+from untappd_scraper.structs.web import WebActivityBeer, WebBeerDetails
+from untappd_scraper.web import id_from_href, parsed_value, slug_from_href
+
+if TYPE_CHECKING:  # pragma: no cover
+    from collections.abc import Iterator
+
+    from requests_html import Element, HTMLResponse
+
+
+class Beer(SimpleRepr):
+    """Untappd beer."""
+
+    def __init__(self, beer_id: int) -> None:
+        """Initiate a Beer object, storing the beer ID and loading details.
+
+        Raises:
+            ValueError: invalid beer ID
+
+        Args:
+            beer_id (int): beer ID
+        """
+        self.beer_id = beer_id
+
+        self._page = get(url_of(beer_id))
+        if not self._page.ok:
+            msg = f"Invalid beer ID {beer_id} ({self._page})"
+            raise ValueError(msg)
+        self._beer_details: WebBeerDetails = beer_details(resp=self._page)
+
+    def __getattr__(self, name: str) -> Any:
+        """Return unknown attributes from beer details.
+
+        Args:
+            name (str): attribute to lookup
+
+        Returns:
+            Any: attribute value
+        """
+        return getattr(self._beer_details, name)
+
+
+# ----- utils -----
+
+
+def url_of(beer_id: int) -> str:
+    """Return the URL for a beer's main page.
+
+    Args:
+        beer_id (int): beer ID
+
+    Returns:
+        str: url to load to get beer's main page
+    """
+    return f"https://untappd.com/beer/{beer_id}"
+
+
+# ----- beer details processing -----
+
+
+def beer_details(resp: HTMLResponse) -> WebBeerDetails:
+    """Parse a user's main page into user details.
+
+    Args:
+        resp (HTMLResponse): beer's main page loaded
+
+    Returns:
+        WebBeerDetails: general beer details
+    """
+    content_el = resp.html.find(".main .content", first=True)
+    description = "".join(
+        content_el.find(".desc .beer-descrption-read-less", first=True).xpath("//div/text()")
+    ).strip()
+
+    return WebBeerDetails(
+        beer_id=id_from_href(content_el.find("a.check", first=True)),
+        name=content_el.find(".name h1", first=True).text,
+        description=description.strip(),
+        brewery=content_el.find(".name .brewery a", first=True).text,
+        brewery_slug=slug_from_href(content_el.find(".name .brewery a", first=True)),
+        style=content_el.find(".name p.style", first=True).text,
+        url=resp.url,
+        global_rating=content_el.find(".details [data-rating]", first=True).attrs[
+            "data-rating"
+        ],
+        num_ratings=parsed_value(
+            "{:d} Rat", content_el.find(".details p.raters", first=True).text
+        ),
+    )
+
+
+def checkin_activity(resp: HTMLResponse) -> Iterator[WebActivityBeer]:
+    """Parse all available recent checkins for a user or in a venue.
+
+    Args:
+        resp (HTMLResponse): user's main page or venue's activity page
+
+    Returns:
+        Iterator[WebActivityBeer]: user's visible recent checkins
+    """
+    return (checkin_details(checkin) for checkin in resp.html.find(".activity .item"))
+
+
+def checkin_details(checkin_item: Element) -> WebActivityBeer:
+    """Extract beer details from a checkin.
+
+    Args:
+        checkin_item (Element): single checkin
+
+    Returns:
+        WebActivityBeer: Interesting details for a beer
+    """
+    user_el, beer_el, brewery_el, location_el = extract_checkin_elements(checkin_item)
+
+    checkin_time_el = checkin_item.find(".bottom .time", first=True)
+    checkin_time = checkin_time_el.attrs.get("data-gregtime", checkin_time_el.text)
+    checkin_time = parse_date(checkin_time)
+    assert checkin_time.tzinfo and checkin_time.tzinfo.utcoffset(checkin_time) is not None, (
+        f"Naive datetime from {checkin_time_el.html=}"
+    )
+
+    purchased_at = checkin_item.find(".purchased a", first=True)
+    try:
+        comment = checkin_item.find("p.comment-text", first=True).text
+    except AttributeError:
+        comment = None
+    serving = checkin_item.find(".serving", first=True)
+
+    data_rating_element = checkin_item.find("[data-rating]", first=True)
+    if data_rating_element:
+        data_rating: float | None = float(data_rating_element.attrs["data-rating"])
+    else:
+        data_rating = None  # pragma: no cover
+
+    try:
+        friends: list[str] | None = [
+            slug_from_href(href) for href in checkin_item.find(".tagged-friends a")
+        ]
+    except AttributeError:  # pragma: no cover
+        friends = None
+
+    return WebActivityBeer(
+        checkin_id=int(checkin_item.attrs["data-checkin-id"]),
+        checkin=checkin_time,
+        user_name=slug_from_href(user_el),
+        name=beer_el.text,
+        beer_id=id_from_href(beer_el),
+        brewery=brewery_el.text,
+        brewery_slug=slug_from_href(brewery_el),
+        location=location_el.text if location_el else None,
+        location_id=id_from_href(location_el) if location_el else None,
+        purchased_at=purchased_at.text if purchased_at else None,
+        purchased_id=id_from_href(purchased_at) if purchased_at else None,
+        comment=comment,
+        serving=serving.text if serving else None,
+        user_rating=data_rating,
+        friends=friends,
+    )
+
+
+def extract_checkin_elements(
+    element: Element,
+) -> tuple[Element, Element, Element, Element | None]:
+    """Extract four linked elements in a checkin.
+
+    Args:
+        element (Element): checkin element
+
+    Raises:
+        ValueError: element passed didn't contain 3-4 <a> tags
+
+    Returns:
+        tuple[Element, Element, Element, Element]: user, beer, brewery, location
+    """
+    elements = element.find(".top .text a")
+
+    if len(elements) == 3:
+        return elements + [None]  # pragma: no cover
+    if len(elements) == 4:
+        return elements
+
+    raise ValueError(
+        f"Wanted 3 or 4 <a> elements (not {len(elements)}) in {element.html}"
+    )  # pragma: no cover

untappd_scraper/html_session.py

@@ -0,0 +1,231 @@
+"""HTML session to be shared across all modules."""
+
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+from typing import TYPE_CHECKING, Final
+
+import ratelim
+import requests
+from requests_cache import CacheMixin
+from requests_html import HTMLResponse, HTMLSession
+from tenacity import (
+    RetryCallState,
+    _utils,
+    retry,
+    retry_base,
+    stop_after_attempt,
+    wait_exponential,
+)
+from tenacity.after import after_log
+from tenacity.wait import wait_base
+
+if TYPE_CHECKING:  # pragma: no cover
+    from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+CACHE_EXPIRY: Final[timedelta] = timedelta(hours=1)
+MAX_GET_MIN1: Final[int] = 40  # max number of web GETs in a minute
+MAX_RETRY_SECS: Final[int] = 180  # never wait more than this for a retry
+MAX_RETRY_ATTEMPTS: Final[int] = 9  # give up after this many retries
+MIN1: Final[int] = 60  # seconds
+
+# Allow user to check these. Don't raise an exception here
+ACCEPTABLE_HTTP_STATUS: Final[frozenset[int]] = frozenset((requests.codes["not_found"],))
+
+
+class CachedHTMLSession(CacheMixin, HTMLSession):  # pyright: ignore[reportIncompatibleMethodOverride]
+    """Session with features from both CachedSession and HTMLSession."""
+
+
+_html_session = CachedHTMLSession(cache_name="html", expire_after=CACHE_EXPIRY)
+
+
+@ratelim.greedy(MAX_GET_MIN1, MIN1)
+def get(url: str, *, emulate_404: bool = False, **kwargs: str) -> requests.Response:
+    """Get a URL.
+
+    Handles too many requests errors, and retries after waiting
+
+    Args:
+        url (str): URL to get
+        emulate_404 (bool): if True, return a 404 response
+        kwargs (dict): extra requests options, eg, params and headers
+
+    Returns:
+        requests.Response: response to get
+    """
+    if emulate_404:
+        url = "https://httpbin.org/status/404"
+    resp = _get(url, **kwargs)
+    logger.debug(
+        "GET %s (%s) received %s\tExpires: %s, Headers: %s",
+        url,
+        kwargs,
+        resp,
+        resp.expires,  # pyright: ignore[reportAttributeAccessIssue]
+        resp.headers,
+    )
+    return resp
+
+
+# ----- Tenacity -----
+
+
+class RetryAfter(wait_base):
+    """Strategy that tries to wait as per Retry-After header.
+
+    Tries to wait for the length specified by the Retry-After header,
+    or the underlying wait / fallback strategy if not.
+    See RFC 6585 § 4.
+    """
+
+    def __init__(self, fallback: wait_base) -> None:
+        """Store fallback strategy in case we can't work out retry wait time.
+
+        Args:
+            fallback (wait_base): fallback wait strategy if no Retry-After found
+        """
+        self.fallback = fallback
+
+    def __call__(self, retry_state: RetryCallState) -> int:  # pragma: no cover
+        """Return seconds to wait until retry.
+
+        Args:
+            retry_state (RetryState): State of retry, with .outcome property
+                storing exception.
+
+        Returns:
+            int: seconds to wait
+        """
+        assert retry_state.outcome
+        exc = retry_state.outcome.exception()
+        if isinstance(exc, requests.HTTPError):
+            retry_after = exc.response.headers.get("Retry-After")
+            logger.debug("Searching response header and found Retry-After of %s", retry_after)
+
+            try:
+                return int(retry_after)  # pyright: ignore[reportArgumentType]
+            except (TypeError, ValueError):
+                return int(self.fallback(retry_state))
+
+        return int(self.fallback(retry_state))
+
+
+def my_before_sleep_log(
+    user_logger: logging.Logger, *, exc_info: bool = False
+) -> Callable[[RetryCallState], None]:
+    """Before call strategy that logs to some logger the attempt.
+
+    Logging level is determined by the number of retries.
+
+    Lifted from Tenacity function and removed hard-coded log level
+
+    Args:
+        user_logger (Logger): logger to use
+        exc_info (bool, optional): Is there an exception. Defaults to False.
+
+    Returns:
+        logging function
+    """
+
+    def log_it(retry_state: RetryCallState) -> None:  # pragma: no cover
+        if retry_state.attempt_number < 1:
+            log_level = logging.DEBUG
+        elif retry_state.attempt_number == 1:
+            log_level = logging.INFO
+        else:
+            log_level = logging.WARNING
+
+        assert retry_state.outcome
+
+        if retry_state.outcome.failed:
+            ex = retry_state.outcome.exception()
+            verb, retry_value = "raised", f"{type(ex).__name__}: {ex}"
+
+            if exc_info and retry_state.outcome:
+                local_exc_info = retry_state.outcome.exception()
+            else:
+                local_exc_info = False
+        else:
+            verb, retry_value = "returned", retry_state.outcome.result()
+            local_exc_info = False  # exc_info does not apply when no exception
+
+        user_logger.log(
+            log_level,
+            "Retrying %s (%s attempt) in %s seconds as it %s %s.",
+            _utils.get_callback_name(retry_state.fn),  # pyright: ignore[reportArgumentType]
+            _utils.to_ordinal(retry_state.attempt_number),
+            retry_state.next_action.sleep,  # pyright: ignore[reportOptionalMemberAccess]
+            verb,
+            retry_value,
+            exc_info=local_exc_info,
+        )
+
+    return log_it
+
+
+def is_throttling_related_exception(excp: Exception) -> bool:  # pragma: no cover
+    """Check is the exception is a requests one and throttling related.
+
+    Args:
+        excp (Exception): exception raised
+
+    Returns:
+        bool: was it a throttle
+    """
+    return (
+        isinstance(excp, requests.HTTPError)
+        and excp.response.status_code == requests.codes.too_many_requests
+    )
+
+
+class RetryIfThrottling(retry_base):
+    """Retry class which only retries if a throttling exception occured.
+
+    From https://www.seelk.co/blog/efficient-client-side-handling-of-api-throttling-in-python-with-tenacity/
+    """
+
+    def __call__(self, retry_state: RetryCallState) -> bool:  # pragma: no cover
+        """Return if the call raised an exception and it's a throttle.
+
+        Args:
+            retry_state (RetryCallState): info about current retry invocation
+
+        Returns:
+            bool: is it throttling related
+        """
+        if (
+            retry_state.outcome
+            and retry_state.outcome.failed
+            and (exception := retry_state.outcome.exception())
+        ):
+            return is_throttling_related_exception(exception)  # pyright: ignore[reportArgumentType]
+        return False
+
+
+# ---- Main part of tenacity retry ----
+
+# Retry too-many-requests after a delay
+
+# - Retries if throttle response received
+# - Checks for Retry-After header and wait that long
+# - If no header, waits expontially longer each retry
+# - logs retries, with increasing severity as attempts increase
+
+
+@retry(
+    reraise=True,
+    retry=RetryIfThrottling(),
+    wait=RetryAfter(fallback=wait_exponential(min=5, max=MAX_RETRY_SECS)),
+    stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
+    before_sleep=my_before_sleep_log(logger),
+    after=after_log(logger, logging.DEBUG),
+)
+def _get(url: str, **kwargs: str) -> HTMLResponse:
+    resp = _html_session.get(url, **kwargs)
+    if resp.status_code not in ACCEPTABLE_HTTP_STATUS:
+        resp.raise_for_status()  # pragma: no cover
+    return resp  # pyright: ignore[reportReturnType]

untappd_scraper/logging_config.py

@@ -0,0 +1,78 @@
+"""Set up logging using the Loguru library."""
+
+from __future__ import annotations
+
+import inspect
+import logging
+import sys
+from pathlib import Path
+from typing import Final
+
+from loguru import logger
+
+# ----- Constants -----
+
+# Log formats
+STANDARD: Final = "[{time:HH:mm:ss}] {level} - {message}"
+DETAIL: Final = "{time} {file:>25}:{line:<4} {level:<8} {message}"
+ROTATION: Final = "1 hour"
+RETENTION: Final = "2 days"
+
+
+def configure_logging(
+    log_filename: str | Path,
+    *,
+    standard_format: str = STANDARD,
+    detail_format: str = DETAIL,
+    log_rotation: str = ROTATION,
+    log_retention: str = RETENTION,
+) -> None:
+    """Setup logging for the application."""
+    # Capture things like Hishel logging
+    intercept_logging()
+    # Replace the default StdErr handler.
+    logger.remove()
+    logger.add(sys.stderr, level="WARNING", format=standard_format)
+
+    log_filename = Path(log_filename)
+    if log_filename.suffix != ".log":
+        log_filename = log_filename.with_suffix(".log")
+
+    # Add a rotating file handler.
+    logger.add(
+        log_filename,
+        level="DEBUG",
+        format=detail_format,
+        rotation=log_rotation,
+        retention=log_retention,
+    )
+
+
+# ----- Interface to the standard logging module -----
+
+
+class InterceptHandler(logging.Handler):
+    """Send logs to Loguru."""
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Emit a log record."""
+        # Get corresponding Loguru level if it exists.
+        level: str | int
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller from where originated the logged message.
+        frame, depth = inspect.currentframe(), 0
+        while frame and (depth == 0 or frame.f_code.co_filename == logging.__file__):
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
+def intercept_logging() -> None:
+    """Intercept standard logging and send it to Loguru."""
+    # Configure the root logger
+    logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)

untappd_scraper/logs.yml

@@ -0,0 +1,59 @@
+# Inspired from https://towardsdatascience.com/
+#   ultimate-guide-to-python-debugging-854dea731e1b
+
+version: 1
+disable_existing_loggers: false
+
+formatters:
+  standard:
+    format: "[%(asctime)s] %(levelname)s - %(message)s"
+    datefmt: "%H:%M:%S"
+  detail:
+    format: "%(asctime)s %(filename)25s:%(lineno)-4d %(levelname)-8s %(message)s"
+
+handlers:
+  console: # handler which will log into stdout
+    class: logging.StreamHandler
+    level: WARNING
+    formatter: standard # Use formatter defined above
+    stream: ext://sys.stdout
+  filesize: # handler which will log to file up to a size
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: detail # Use formatter defined above
+    filename: ext://os.devnull
+    maxBytes: 10485760 # 10 Mb
+    backupCount: 10
+    encoding: utf8
+  filetime: # handler which will switch files regularly
+    class: logging.handlers.TimedRotatingFileHandler
+    level: INFO
+    formatter: detail # Use formatter defined above
+    filename: ext://os.devnull
+    when: W0
+    # interval: 0
+    # maxBytes:  10485760
+    # backupCount:
+    encoding: utf8
+  email: # email handler for error messages
+    class: logging.handlers.SMTPHandler
+    level: ERROR
+    formatter: standard # Use formatter defined above
+    mailhost: tbd # ext://util.config.log.mailhost
+    # credentials: ext://util.config.log.credentials
+    # secure: ext://util.config.log.secure
+    fromaddr: tbd # ext://util.config.log.fromaddr
+    toaddrs: tbd # ext://util.config.log.toaddrs
+    # - support_team@domain.tld
+    # - dev_team@domain.tld
+    subject: tbd #  ext://util.config.log.subject # Houston, we have a problem.
+
+root: # Loggers are organized in hierarchy - this is the root logger config
+  level: DEBUG
+  handlers: [console] # , file, email] # Attaches both handler defined above
+
+# loggers: # Defines descendants of root logger
+#   mymodule: # Logger for "mymodule"
+#     level: INFO
+#     handlers: [file] # Will only use "file" handler defined above
+#     propagate: no # Will not propagate logs to "root" logger

untappd_scraper/structs/mixins.py

@@ -0,0 +1,63 @@
+"""Mixin classes."""
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class HasBeerId(Protocol):
+    """Check a class has a beer id."""
+
+    beer_id: int
+
+
+class HasVenueId(Protocol):
+    """Check a class has a venue id."""
+
+    venue_id: int
+
+
+class IdStrMixin:
+    """Provide string versions of integer IDs."""
+
+    @property
+    def beer_id_str(self: HasBeerId) -> str:
+        """Return str version of Beer ID.
+
+        JSON keys are str but dict can be int. Can get dupe JSON keys.
+
+        Returns:
+            str: string version of beer ID
+        """
+        return str(self.beer_id)
+
+    @property
+    def venue_id_str(self: HasVenueId) -> str:
+        """Return str version of Venue ID.
+
+        JSON keys are str but dict can be int. Can get dupe JSON keys.
+
+        Returns:
+            str: string version of venue ID
+        """
+        return str(self.venue_id)
+
+
+class HasBeerDetails(Protocol):
+    """Check if a class has beer details."""
+
+    name: str
+    brewery: str
+    style: str
+    global_rating: float
+
+
+class BeerStrMixin:
+    """Provide a nice description of a beer."""
+
+    def __str__(self: HasBeerDetails) -> str:
+        """Create a summary description of a beer.
+
+        Returns:
+            str: beer description
+        """
+        return f"{self.name} by {self.brewery}, {self.style} ({self.global_rating})"

untappd_scraper/structs/other.py

@@ -0,0 +1,27 @@
+"""Structures used to represent data not scraped."""
+from __future__ import annotations
+
+from dataclasses import astuple, dataclass
+from functools import lru_cache
+
+from haversine import haversine
+
+
+@dataclass(frozen=True)
+class Location:
+    """Store latitude and longiture and calculate haversine distance between them."""
+
+    lat: float
+    lng: float
+
+    @lru_cache
+    def distance_from(self, other: Location) -> float:
+        """Return km between two Location objects.
+
+        Args:
+            other (Location): where to measure distance to
+
+        Returns:
+            float: distance between points in km
+        """
+        return haversine(astuple(self), astuple(other))