python-ballchasing 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

ballchasing/__init__.py

@@ -49,6 +49,7 @@ except Exception:
 
 # Import the main classes and constants to make them easily accessible to users.
 from .api import BallchasingApi
+from .rate_limit import RateLimiter, RateLimitCategory, rate_limited
 
 Api = BallchasingApi  # For importing like `import ballchasing; api = ballchasing.Api`
 

ballchasing/api.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor
@@ -6,18 +7,30 @@ from pathlib import Path
 from typing import BinaryIO, Iterator
 from urllib.parse import parse_qs, urlparse
 
-from requests import sessions, Response, ConnectionError
+from requests import sessions, Response
+from requests.exceptions import ConnectionError, Timeout
 
 from ballchasing.constants import GroupSortBy, SortDir, AnySeason, AnyRank, AnyReplaySortBy, AnySortDir, \
     AnyVisibility, AnyGroupSortBy, AnyPlayerIdentification, AnyTeamIdentification, AnyMatchResult, NoneOrMore
+from ballchasing.rate_limit import RateLimiter, RateLimitCategory, resolve_category, rate_limited
 from ballchasing.typed import DeepReplay, ShallowReplay, DeepGroup, ShallowGroup
 from .typed.shared import BaseGroup, BasicGroup
 from .util.dates import to_rfc3339
 from .util.iterators import deduplicate as deduplicator
 from .util.stats import parse_replay_stats
 
+logger = logging.getLogger("ballchasing")
+
 DEFAULT_URL = "https://ballchasing.com/api"
 
+PATRON_RATE_LIMIT_SLEEP_TIMES = {
+    "regular": 3600 / 1000,
+    "gold": 3600 / 2000,
+    "diamond": 3600 / 5000,
+    "champion": 1 / 8,
+    "gc": 1 / 16,
+}
+
 
 class BallchasingApi:
     """
@@ -33,6 +46,7 @@ class BallchasingApi:
             base_url: str | None = None,
             do_initial_ping: bool = True,
             typed: bool = False,
+            proactive_rate_limit: bool = False,
     ):
         """
 
@@ -40,6 +54,7 @@ class BallchasingApi:
         :param sleep_time_on_rate_limit: seconds to wait after being rate limited.
                                          Default value is calculated depending on patron type.
         :param print_on_rate_limit: whether or not to print upon rate limits.
+        :param proactive_rate_limit: whether to pace requests proactively before hitting limits.
         """
         self.auth_key = auth_key
         self._session = sessions.Session()
@@ -48,18 +63,40 @@ class BallchasingApi:
         self.base_url = DEFAULT_URL if base_url is None else base_url
         if do_initial_ping:
             self.ping()
-        if sleep_time_on_rate_limit is None:
-            self.sleep_time_on_rate_limit = {
-                "regular": 3600 / 1000,
-                "gold": 3600 / 2000,
-                "diamond": 3600 / 5000,
-                "champion": 1 / 8,
-                "gc": 1 / 16
-            }.get(self.patron_type or "regular")
-        else:
-            self.sleep_time_on_rate_limit = sleep_time_on_rate_limit
+        self._sleep_time_on_rate_limit = sleep_time_on_rate_limit
         self.print_on_rate_limit = print_on_rate_limit
         self.typed = typed
+        self.rate_limiter = RateLimiter(
+            patron_type_getter=lambda: self._ping_result.get("type") if self._ping_result else None,
+            proactive=proactive_rate_limit,
+        )
+
+    @property
+    def sleep_time_on_rate_limit(self) -> float:
+        if self._sleep_time_on_rate_limit is not None:
+            return self._sleep_time_on_rate_limit
+        if self._ping_result is not None:
+            return PATRON_RATE_LIMIT_SLEEP_TIMES.get(self.patron_type or "regular", 3600 / 1000)
+        return PATRON_RATE_LIMIT_SLEEP_TIMES["regular"]
+
+    @sleep_time_on_rate_limit.setter
+    def sleep_time_on_rate_limit(self, value: float | None):
+        self._sleep_time_on_rate_limit = value
+
+    @property
+    def last_rate_limited_at(self) -> float | None:
+        """Timestamp (time.time()) of the most recent 429 response, or None if never rate-limited."""
+        return self.rate_limiter.last_rate_limited_at
+
+    @property
+    def is_rate_limited(self) -> bool:
+        """True if the instance is currently waiting out a 429 rate-limit cooldown."""
+        return self.rate_limiter.is_rate_limited
+
+    @property
+    def rate_limit_stats(self) -> dict:
+        """Detailed rate-limiting statistics across all categories."""
+        return self.rate_limiter.get_stats()
 
     @property
     def steam_name(self):
@@ -89,49 +126,71 @@ class BallchasingApi:
             self,
             url_or_endpoint: str,
             method: str,
+            allow_status_codes: tuple[int, ...] = (),
+            category: RateLimitCategory | str | None = None,
             **params
     ) -> Response:
         """
         Helper method for all requests.
 
-        :param url: url or endpoint for request.
+        :param url_or_endpoint: url or endpoint for request.
         :param method: the method to use.
-        :param params: parameters for GET request.
+        :param allow_status_codes: additional HTTP status codes to treat as successful responses.
+        :param category: rate limit category for this request.
+        :param params: parameters for request (e.g. params, json, files, stream).
         :return: the request result.
         :raises ConnectionError: if the connection fails after max retries.
-        :raises HTTPError: if the request fails with a status code other than 2xx or 429.
+        :raises HTTPError: if the request fails with a status code other than 2xx, 429, or allow_status_codes.
         """
+        cat = RateLimitCategory(category) if category else resolve_category(url_or_endpoint)
         headers = {"Authorization": self.auth_key}
         url = f"{self.base_url}{url_or_endpoint}" if url_or_endpoint.startswith("/") else url_or_endpoint
         max_retries = 8
         retries = 0
+        rate_limit_retries = 0
+
+        # Wait if global cooldown is active or if proactive pacing applies
+        self.rate_limiter.before_request(cat)
+
         while True:
             try:
                 r: Response = self._session.request(method=method, url=url, headers=headers, **params)
-                if 200 <= r.status_code < 300:
+                if 200 <= r.status_code < 300 or r.status_code in allow_status_codes:
                     return r
                 elif r.status_code == 429:
                     self.rate_limit_count += 1
-                    if self.print_on_rate_limit:
-                        print(f"Rate limited at {url} ({self.rate_limit_count} total rate limits)")
+                    rate_limit_retries += 1
                     retry_after = r.headers.get("Retry-After", '0')
-                    retry_after = int(retry_after) if retry_after.isdigit() else None
-                    if retry_after:  # integer > 0
-                        time.sleep(retry_after)
-                    elif self.sleep_time_on_rate_limit:
-                        time.sleep(self.sleep_time_on_rate_limit)
-                elif r.status_code == 504:
-                    raise ConnectionError("Gateway Timeout. The server did not respond in time.")
+                    retry_after = float(retry_after) if retry_after.isdigit() else None
+                    wait_time = self.rate_limiter.record_429(
+                        cat,
+                        retry_after=retry_after,
+                        fallback_sleep_time=self._sleep_time_on_rate_limit,
+                        retries=rate_limit_retries,
+                    )
+
+                    if self.print_on_rate_limit:
+                        logger.warning(
+                            f"Rate limited at {url} [{cat.value}] ({self.rate_limit_count} total rate limits), retrying in {wait_time:.1f}s..."
+                        )
+                    else:
+                        logger.debug(
+                            f"Rate limited at {url} [{cat.value}] ({self.rate_limit_count} total rate limits), retrying in {wait_time:.1f}s..."
+                        )
+                    time.sleep(wait_time)
+                elif r.status_code in (502, 503, 504):
+                    raise ConnectionError(f"Server error {r.status_code}: {r.reason or 'Transient gateway error'}")
                 else:
-                    r.raise_for_status()  # Raise an error for any other status code'
-            except ConnectionError as e:
+                    r.raise_for_status()  # Raise an error for any other status code
+            except (ConnectionError, Timeout) as e:
                 if retries >= max_retries - 1:
                     raise e
                 s = 2 ** retries
-                print(f"Connection error, trying again in {s} seconds...")
+                logger.warning(f"Connection or timeout error ({e}), trying again in {s} seconds...")
                 time.sleep(s)
                 retries += 1
 
+    @rate_limited(RateLimitCategory.PING)
     def ping(self) -> dict:
         """
         Use this API to:
@@ -142,7 +201,7 @@ class BallchasingApi:
         This method runs automatically at initialization and the steam name and id as well as patron type are stored.
         :return: ping response.
         """
-        result = self._request("/", "GET").json()
+        result = self._request("/", "GET", category=RateLimitCategory.PING).json()
         self._ping_result = result
         return result
 
@@ -153,10 +212,12 @@ class BallchasingApi:
         # consumer processing items.  When prefetch=False (e.g. deep mode
         # where the consumer also makes API calls), the next request is
         # submitted *after* yielding to avoid doubling the request rate.
-        remaining = params["count"]
+        remaining = params.get("count", 0)
+        if remaining <= 0:
+            return
 
         def fetch_page(p):
-            return self._request(url, "GET", params=p).json()
+            return self._request(url, "GET", params=p, category=RateLimitCategory.LIST).json()
 
         with ThreadPoolExecutor(max_workers=1) as executor:
             params["count"] = min(remaining, 200)
@@ -164,15 +225,19 @@ class BallchasingApi:
 
             while remaining > 0:
                 d = future.result()
-                batch = d["list"][:min(remaining, 200)]
+                batch = d.get("list", [])[:min(remaining, 200)]
                 remaining -= len(batch)
 
-                has_next = "next" in d and remaining > 0
+                next_url = d.get("next")
+                has_next = bool(next_url) and remaining > 0
                 if has_next:
-                    next_url = d["next"]
-                    params["after"] = parse_qs(urlparse(next_url).query)["after"][0]
+                    after_param = parse_qs(urlparse(next_url).query).get("after", [None])[0]
+                    if after_param is not None:
+                        params["after"] = after_param
+                    else:
+                        has_next = False
                     params["count"] = min(remaining, 200)
-                    if prefetch:
+                    if prefetch and has_next:
                         future = executor.submit(fetch_page, dict(params))
 
                 yield from batch
@@ -182,6 +247,7 @@ class BallchasingApi:
                 if not prefetch:
                     future = executor.submit(fetch_page, dict(params))
 
+    @rate_limited(RateLimitCategory.LIST)
     def get_replays(
             self,
             *,
@@ -238,8 +304,8 @@ class BallchasingApi:
                               RFC3339 format, e.g. '2020-01-02T15:00:05+01:00'
         :param count: returns at most count replays. Since the implementation uses an iterator it supports iterating
                       past the limit of 200 set by the API
-        :param sort_by: sort replays according the selected field
-        :param sort_dir: sort direction
+        :param sort_by: sort replays according the selected field ('replay-date', 'upload-date')
+        :param sort_dir: sort direction ('asc', 'desc')
         :param deep: whether to get full stats for each replay (will be much slower).
         :param typed: whether to return a typed object (default is self.typed).
         :param deduplicate: whether to deduplicate replays that seem to be the same game.
@@ -262,11 +328,11 @@ class BallchasingApi:
             disable_prefetch = deep
 
         iterator = self._iterable_from_request(url, params, prefetch=not disable_prefetch)
-        if deep:
-            iterator = (self.get_replay(r["id"]) for r in iterator)
         if deduplicate:
             # Deep replays have match and replay IDs to deduplicate with. For shallow replays we check dates.
             iterator = deduplicator(iterator, check_dates=not deep)
+        if deep:
+            iterator = (self.get_replay(r["id"]) for r in iterator)
         if typed:
             if deep:
                 iterator = (DeepReplay(**r) for r in iterator)
@@ -274,6 +340,7 @@ class BallchasingApi:
                 iterator = (ShallowReplay(**r) for r in iterator)
         yield from iterator
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_replay(self, replay_id: str, *, typed: bool | None = None) -> dict | DeepReplay:
         """
         Retrieve a given replay’s details and stats.
@@ -282,43 +349,82 @@ class BallchasingApi:
         :param typed: whether to return a typed object (default is self.typed).
         :return: the result of the GET request.
         """
-        result = self._request(f"/replays/{replay_id}", "GET").json()
+        result = self._request(f"/replays/{replay_id}", "GET", category=RateLimitCategory.CORE).json()
         if typed is None:
             typed = self.typed
         if typed:
             result = DeepReplay(**result)
         return result
 
-    def patch_replay(self, replay_id: str, **params) -> None:
+    @rate_limited(RateLimitCategory.CORE)
+    def patch_replay(
+            self,
+            replay_id: str,
+            *,
+            title: str | None = None,
+            visibility: AnyVisibility | None = None,
+            group: str | None = None,
+            **params
+    ) -> None:
         """
-        This endpoint can patch one or more fields of the specified replay
+        This endpoint can patch one or more fields of the specified replay.
 
         :param replay_id: the replay id.
-        :param params: parameters for the PATCH request.
-        """
-        self._request(f"/replays/{replay_id}", "PATCH", json=params)
-
+        :param title: new title for the replay.
+        :param visibility: new visibility ('public', 'unlisted', 'private').
+        :param group: group id to assign, or "" to unassign replay from group.
+        :param params: additional parameters for the PATCH request.
+        """
+        payload = dict(params)
+        if title is not None:
+            payload["title"] = title
+        if visibility is not None:
+            payload["visibility"] = visibility
+        if group is not None:
+            payload["group"] = group
+        self._request(f"/replays/{replay_id}", "PATCH", json=payload, category=RateLimitCategory.CORE)
+
+    @rate_limited(RateLimitCategory.CORE)
     def upload_replay(
             self,
             replay_file: str | Path | BinaryIO,
             *,
             visibility: AnyVisibility | None = None,
-            group: str | None = None
+            group: str | None = None,
+            raise_on_duplicate: bool = False,
     ) -> dict:
         """
         Use this API to upload a replay file to ballchasing.com.
 
         :param replay_file: replay file to upload. Can be a file path (str or Path) or a file-like object.
-        :param visibility: to set the visibility of the uploaded replay.
+        :param visibility: to set the visibility of the uploaded replay ('public', 'unlisted', 'private').
         :param group: to upload the replay to an existing group.
-        :return: the result of the POST request.
+        :param raise_on_duplicate: if True, raise an HTTPError when a 409 Conflict (duplicate replay)
+                                   is returned. If False (default), returns the API response dict with the
+                                   existing replay id.
+        :return: the result of the POST request (dict containing replay 'id').
         """
         if isinstance(replay_file, (str, Path)):
             with open(replay_file, "rb") as f:
-                return self.upload_replay(f, visibility=visibility, group=group)
-        return self._request(f"/v2/upload", "POST", files={"file": replay_file},
-                             params={"group": group, "visibility": visibility}).json()
-
+                return self.upload_replay(
+                    f,
+                    visibility=visibility,
+                    group=group,
+                    raise_on_duplicate=raise_on_duplicate,
+                )
+        r = self._request(
+            "/v2/upload",
+            "POST",
+            files={"file": replay_file},
+            params={"group": group, "visibility": visibility},
+            allow_status_codes=(409,),
+            category=RateLimitCategory.CORE,
+        )
+        if r.status_code == 409 and raise_on_duplicate:
+            r.raise_for_status()
+        return r.json()
+
+    @rate_limited(RateLimitCategory.CORE)
     def delete_replay(self, replay_id: str) -> None:
         """
         This endpoint deletes the specified replay.
@@ -326,8 +432,9 @@ class BallchasingApi:
 
         :param replay_id: the replay id.
         """
-        self._request(f"/replays/{replay_id}", "DELETE")
+        self._request(f"/replays/{replay_id}", "DELETE", category=RateLimitCategory.CORE)
 
+    @rate_limited(RateLimitCategory.LIST)
     def get_groups(
             self,
             *,
@@ -356,14 +463,14 @@ class BallchasingApi:
                               RFC3339 format, e.g. 2020-01-02T15:00:05+01:00
         :param count: returns at most count groups. Since the implementation uses an iterator it supports iterating
                       past the limit of 200 set by the API
-        :param sort_by: Sort groups according the selected field.
-        :param sort_dir: Sort direction.
+        :param sort_by: Sort groups according the selected field ('created', 'name').
+        :param sort_dir: Sort direction ('asc', 'desc').
         :param deep: whether to get full stats for each group (will be much slower).
         :param typed: whether to return a typed object (default is self.typed).
         :param disable_prefetch: whether to disable prefetching.
         :return: an iterator over the groups returned by the API.
         """
-        url = f"{self.base_url}/groups/"
+        url = f"{self.base_url}/groups"
         params = {"name": name, "creator": creator, "group": group, "created-before": to_rfc3339(created_before),
                   "created-after": to_rfc3339(created_after), "count": count, "sort-by": sort_by, "sort-dir": sort_dir}
 
@@ -379,6 +486,7 @@ class BallchasingApi:
             iterator = (ShallowGroup(**g) for g in iterator)
         yield from iterator
 
+    @rate_limited(RateLimitCategory.CORE)
     def create_group(
             self,
             *,
@@ -391,7 +499,7 @@ class BallchasingApi:
         Use this API to create a new replay group.
 
         :param name: the new group name.
-        :param player_identification: how to identify the same player across multiple replays.
+        :param player_identification: how to identify the same player across multiple replays ('by-id', 'by-name').
                                       Some tournaments (e.g. RLCS) make players use a pool of generic Steam accounts,
                                       meaning the same player could end up using 2 different accounts in 2 series.
                                       That's when the `by-name` comes in handy
@@ -399,13 +507,19 @@ class BallchasingApi:
                                     Set to `by-distinct-players` if teams have a fixed roster of players for
                                     every single game. In some tournaments/leagues, teams allow player rotations,
                                     or a sub can replace another player, in which case use `by-player-clusters`.
-        :param parent: if set,the new group will be created as a child of the specified group
+        :param parent: if set, the new group will be created as a child of the specified group
         :return: the result of the POST request.
         """
-        json = {"name": name, "player_identification": player_identification,
-                "team_identification": team_identification, "parent": parent}
-        return self._request(f"/groups", "POST", json=json).json()
+        json = {
+            "name": name,
+            "player_identification": player_identification,
+            "team_identification": team_identification,
+        }
+        if parent is not None:
+            json["parent"] = parent
+        return self._request(f"/groups", "POST", json=json, category=RateLimitCategory.CORE).json()
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_group(
             self,
             group_id: str,
@@ -419,22 +533,46 @@ class BallchasingApi:
         :param typed: whether to return a typed object (default is self.typed).
         :return: the group info with stats.
         """
-        result = self._request(f"/groups/{group_id}", "GET").json()
+        result = self._request(f"/groups/{group_id}", "GET", category=RateLimitCategory.CORE).json()
         if typed is None:
             typed = self.typed
         if typed:
             result = DeepGroup(**result)
         return result
 
-    def patch_group(self, group_id: str, **params) -> None:
+    @rate_limited(RateLimitCategory.CORE)
+    def patch_group(
+            self,
+            group_id: str,
+            *,
+            player_identification: AnyPlayerIdentification | None = None,
+            team_identification: AnyTeamIdentification | None = None,
+            parent: str | None = None,
+            shared: bool | None = None,
+            **params
+    ) -> None:
         """
         This endpoint can patch one or more fields of the specified group.
 
         :param group_id: the group id
-        :param params: parameters for the PATCH request.
-        """
-        self._request(f"/groups/{group_id}", "PATCH", json=params)
-
+        :param player_identification: how to identify the same player across multiple replays ('by-id', 'by-name').
+        :param team_identification: how to identify the same team ('by-distinct-players', 'by-player-clusters').
+        :param parent: parent group id to assign, or "" to unparent.
+        :param shared: whether to mark the group as shared.
+        :param params: additional parameters for the PATCH request.
+        """
+        payload = dict(params)
+        if player_identification is not None:
+            payload["player_identification"] = player_identification
+        if team_identification is not None:
+            payload["team_identification"] = team_identification
+        if parent is not None:
+            payload["parent"] = parent
+        if shared is not None:
+            payload["shared"] = shared
+        self._request(f"/groups/{group_id}", "PATCH", json=payload, category=RateLimitCategory.CORE)
+
+    @rate_limited(RateLimitCategory.CORE)
     def delete_group(self, group_id: str) -> None:
         """
         This endpoint deletes the specified group.
@@ -442,7 +580,7 @@ class BallchasingApi:
 
         :param group_id: the group id.
         """
-        self._request(f"/groups/{group_id}", "DELETE")
+        self._request(f"/groups/{group_id}", "DELETE", category=RateLimitCategory.CORE)
 
     def get_group_replays(
             self,
@@ -490,22 +628,25 @@ class BallchasingApi:
         for replay in self.get_replays(group_id=group_id, deep=deep, typed=typed):
             yield [group_id], replay
 
-    def download_replay(self, replay_id: str, path: str):
+    @rate_limited(RateLimitCategory.DOWNLOAD)
+    def download_replay(self, replay_id: str, path: str | Path) -> Path:
         """
         Download a replay file.
 
         :param replay_id: the replay id.
         :param path: the path to download the replay to. Can be a file path or a directory.
+        :return: the Path to the downloaded file.
         """
-        r = self._request(f"/replays/{replay_id}/file", "GET")
-        if os.path.isdir(path):
-            # If path is a directory, use the replay id as the filename
-            filename = f"{replay_id}.replay"
-            path = os.path.join(path, filename)
-        with open(path, "wb") as f:
-            f.write(r.content)
+        dest = Path(path)
+        if dest.is_dir():
+            dest = dest / f"{replay_id}.replay"
+        r = self._request(f"/replays/{replay_id}/file", "GET", stream=True, category=RateLimitCategory.DOWNLOAD)
+        with open(dest, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+        return dest
 
-    def download_group(self, group_id: str, folder: str, *, keep_tree_structure=True):
+    def download_group(self, group_id: str, folder: str | Path, *, keep_tree_structure=True):
         """
         Download an entire group.
 
@@ -513,22 +654,24 @@ class BallchasingApi:
         :param folder: the folder in which to create the group folder.
         :param keep_tree_structure: whether to create new folders for child groups.
         """
-        folder = os.path.join(folder, group_id)
+        folder = Path(folder) / group_id
         if keep_tree_structure:
-            os.makedirs(folder, exist_ok=True)
+            folder.mkdir(parents=True, exist_ok=True)
             for child_group in self.get_groups(group=group_id):
                 self.download_group(child_group["id"], folder, keep_tree_structure=True)
             for replay in self.get_replays(group_id=group_id):
                 self.download_replay(replay["id"], folder)
         else:
+            folder.mkdir(parents=True, exist_ok=True)
             for replay in self.get_group_replays(group_id):
                 self.download_replay(replay["id"], folder)
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_maps(self):
         """
         Use this API to get the list of map codes to map names (map as in stadium).
         """
-        res = self._request("/maps", "GET").json()
+        res = self._request("/maps", "GET", category=RateLimitCategory.CORE).json()
         return res
 
     def get_stats(self, replay: dict | str):