python-ballchasing 0.5.2__tar.gz → 0.5.3__tar.gz

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-ballchasing
-Version: 0.5.2
+Version: 0.5.3
 Summary: A Python wrapper around the Ballchasing API
 Author-email: Rolv-Arild Braaten <rolv_arild@hotmail.com>
 License: MIT License

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/ballchasing/__init__.py

@@ -49,6 +49,7 @@ except Exception:
 
 # Import the main classes and constants to make them easily accessible to users.
 from .api import BallchasingApi
+from .rate_limit import RateLimiter, RateLimitCategory, rate_limited
 
 Api = BallchasingApi  # For importing like `import ballchasing; api = ballchasing.Api`
 

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/ballchasing/api.py

@@ -12,6 +12,7 @@ from requests.exceptions import ConnectionError, Timeout
 
 from ballchasing.constants import GroupSortBy, SortDir, AnySeason, AnyRank, AnyReplaySortBy, AnySortDir, \
     AnyVisibility, AnyGroupSortBy, AnyPlayerIdentification, AnyTeamIdentification, AnyMatchResult, NoneOrMore
+from ballchasing.rate_limit import RateLimiter, RateLimitCategory, resolve_category, rate_limited
 from ballchasing.typed import DeepReplay, ShallowReplay, DeepGroup, ShallowGroup
 from .typed.shared import BaseGroup, BasicGroup
 from .util.dates import to_rfc3339
@@ -45,6 +46,7 @@ class BallchasingApi:
             base_url: str | None = None,
             do_initial_ping: bool = True,
             typed: bool = False,
+            proactive_rate_limit: bool = False,
     ):
         """
 
@@ -52,6 +54,7 @@ class BallchasingApi:
         :param sleep_time_on_rate_limit: seconds to wait after being rate limited.
                                          Default value is calculated depending on patron type.
         :param print_on_rate_limit: whether or not to print upon rate limits.
+        :param proactive_rate_limit: whether to pace requests proactively before hitting limits.
         """
         self.auth_key = auth_key
         self._session = sessions.Session()
@@ -63,6 +66,10 @@ class BallchasingApi:
         self._sleep_time_on_rate_limit = sleep_time_on_rate_limit
         self.print_on_rate_limit = print_on_rate_limit
         self.typed = typed
+        self.rate_limiter = RateLimiter(
+            patron_type_getter=lambda: self._ping_result.get("type") if self._ping_result else None,
+            proactive=proactive_rate_limit,
+        )
 
     @property
     def sleep_time_on_rate_limit(self) -> float:
@@ -76,6 +83,21 @@ class BallchasingApi:
     def sleep_time_on_rate_limit(self, value: float | None):
         self._sleep_time_on_rate_limit = value
 
+    @property
+    def last_rate_limited_at(self) -> float | None:
+        """Timestamp (time.time()) of the most recent 429 response, or None if never rate-limited."""
+        return self.rate_limiter.last_rate_limited_at
+
+    @property
+    def is_rate_limited(self) -> bool:
+        """True if the instance is currently waiting out a 429 rate-limit cooldown."""
+        return self.rate_limiter.is_rate_limited
+
+    @property
+    def rate_limit_stats(self) -> dict:
+        """Detailed rate-limiting statistics across all categories."""
+        return self.rate_limiter.get_stats()
+
     @property
     def steam_name(self):
         if self._ping_result is None:
@@ -104,45 +126,58 @@ class BallchasingApi:
             self,
             url_or_endpoint: str,
             method: str,
+            allow_status_codes: tuple[int, ...] = (),
+            category: RateLimitCategory | str | None = None,
             **params
     ) -> Response:
         """
         Helper method for all requests.
 
-        :param url: url or endpoint for request.
+        :param url_or_endpoint: url or endpoint for request.
         :param method: the method to use.
-        :param params: parameters for GET request.
+        :param allow_status_codes: additional HTTP status codes to treat as successful responses.
+        :param category: rate limit category for this request.
+        :param params: parameters for request (e.g. params, json, files, stream).
         :return: the request result.
         :raises ConnectionError: if the connection fails after max retries.
-        :raises HTTPError: if the request fails with a status code other than 2xx or 429.
+        :raises HTTPError: if the request fails with a status code other than 2xx, 429, or allow_status_codes.
         """
+        cat = RateLimitCategory(category) if category else resolve_category(url_or_endpoint)
         headers = {"Authorization": self.auth_key}
         url = f"{self.base_url}{url_or_endpoint}" if url_or_endpoint.startswith("/") else url_or_endpoint
         max_retries = 8
         retries = 0
         rate_limit_retries = 0
+
+        # Wait if global cooldown is active or if proactive pacing applies
+        self.rate_limiter.before_request(cat)
+
         while True:
             try:
                 r: Response = self._session.request(method=method, url=url, headers=headers, **params)
-                if 200 <= r.status_code < 300:
+                if 200 <= r.status_code < 300 or r.status_code in allow_status_codes:
                     return r
                 elif r.status_code == 429:
                     self.rate_limit_count += 1
-                    if self.print_on_rate_limit:
-                        logger.warning(f"Rate limited at {url} ({self.rate_limit_count} total rate limits)")
-                    else:
-                        logger.debug(f"Rate limited at {url} ({self.rate_limit_count} total rate limits)")
                     rate_limit_retries += 1
-                    if rate_limit_retries > max_retries:
-                        r.raise_for_status()
                     retry_after = r.headers.get("Retry-After", '0')
-                    retry_after = int(retry_after) if retry_after.isdigit() else None
-                    if retry_after:  # integer > 0
-                        time.sleep(retry_after)
-                    elif self.sleep_time_on_rate_limit:
-                        time.sleep(self.sleep_time_on_rate_limit)
+                    retry_after = float(retry_after) if retry_after.isdigit() else None
+                    wait_time = self.rate_limiter.record_429(
+                        cat,
+                        retry_after=retry_after,
+                        fallback_sleep_time=self._sleep_time_on_rate_limit,
+                        retries=rate_limit_retries,
+                    )
+
+                    if self.print_on_rate_limit:
+                        logger.warning(
+                            f"Rate limited at {url} [{cat.value}] ({self.rate_limit_count} total rate limits), retrying in {wait_time:.1f}s..."
+                        )
                     else:
-                        time.sleep(max(1.0, float(2 ** (rate_limit_retries - 1))))
+                        logger.debug(
+                            f"Rate limited at {url} [{cat.value}] ({self.rate_limit_count} total rate limits), retrying in {wait_time:.1f}s..."
+                        )
+                    time.sleep(wait_time)
                 elif r.status_code in (502, 503, 504):
                     raise ConnectionError(f"Server error {r.status_code}: {r.reason or 'Transient gateway error'}")
                 else:
@@ -155,6 +190,7 @@ class BallchasingApi:
                 time.sleep(s)
                 retries += 1
 
+    @rate_limited(RateLimitCategory.PING)
     def ping(self) -> dict:
         """
         Use this API to:
@@ -165,7 +201,7 @@ class BallchasingApi:
         This method runs automatically at initialization and the steam name and id as well as patron type are stored.
         :return: ping response.
         """
-        result = self._request("/", "GET").json()
+        result = self._request("/", "GET", category=RateLimitCategory.PING).json()
         self._ping_result = result
         return result
 
@@ -181,7 +217,7 @@ class BallchasingApi:
             return
 
         def fetch_page(p):
-            return self._request(url, "GET", params=p).json()
+            return self._request(url, "GET", params=p, category=RateLimitCategory.LIST).json()
 
         with ThreadPoolExecutor(max_workers=1) as executor:
             params["count"] = min(remaining, 200)
@@ -211,6 +247,7 @@ class BallchasingApi:
                 if not prefetch:
                     future = executor.submit(fetch_page, dict(params))
 
+    @rate_limited(RateLimitCategory.LIST)
     def get_replays(
             self,
             *,
@@ -267,8 +304,8 @@ class BallchasingApi:
                               RFC3339 format, e.g. '2020-01-02T15:00:05+01:00'
         :param count: returns at most count replays. Since the implementation uses an iterator it supports iterating
                       past the limit of 200 set by the API
-        :param sort_by: sort replays according the selected field
-        :param sort_dir: sort direction
+        :param sort_by: sort replays according the selected field ('replay-date', 'upload-date')
+        :param sort_dir: sort direction ('asc', 'desc')
         :param deep: whether to get full stats for each replay (will be much slower).
         :param typed: whether to return a typed object (default is self.typed).
         :param deduplicate: whether to deduplicate replays that seem to be the same game.
@@ -303,6 +340,7 @@ class BallchasingApi:
                 iterator = (ShallowReplay(**r) for r in iterator)
         yield from iterator
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_replay(self, replay_id: str, *, typed: bool | None = None) -> dict | DeepReplay:
         """
         Retrieve a given replay’s details and stats.
@@ -311,43 +349,82 @@ class BallchasingApi:
         :param typed: whether to return a typed object (default is self.typed).
         :return: the result of the GET request.
         """
-        result = self._request(f"/replays/{replay_id}", "GET").json()
+        result = self._request(f"/replays/{replay_id}", "GET", category=RateLimitCategory.CORE).json()
         if typed is None:
             typed = self.typed
         if typed:
             result = DeepReplay(**result)
         return result
 
-    def patch_replay(self, replay_id: str, **params) -> None:
+    @rate_limited(RateLimitCategory.CORE)
+    def patch_replay(
+            self,
+            replay_id: str,
+            *,
+            title: str | None = None,
+            visibility: AnyVisibility | None = None,
+            group: str | None = None,
+            **params
+    ) -> None:
         """
-        This endpoint can patch one or more fields of the specified replay
+        This endpoint can patch one or more fields of the specified replay.
 
         :param replay_id: the replay id.
-        :param params: parameters for the PATCH request.
-        """
-        self._request(f"/replays/{replay_id}", "PATCH", json=params)
-
+        :param title: new title for the replay.
+        :param visibility: new visibility ('public', 'unlisted', 'private').
+        :param group: group id to assign, or "" to unassign replay from group.
+        :param params: additional parameters for the PATCH request.
+        """
+        payload = dict(params)
+        if title is not None:
+            payload["title"] = title
+        if visibility is not None:
+            payload["visibility"] = visibility
+        if group is not None:
+            payload["group"] = group
+        self._request(f"/replays/{replay_id}", "PATCH", json=payload, category=RateLimitCategory.CORE)
+
+    @rate_limited(RateLimitCategory.CORE)
     def upload_replay(
             self,
             replay_file: str | Path | BinaryIO,
             *,
             visibility: AnyVisibility | None = None,
-            group: str | None = None
+            group: str | None = None,
+            raise_on_duplicate: bool = False,
     ) -> dict:
         """
         Use this API to upload a replay file to ballchasing.com.
 
         :param replay_file: replay file to upload. Can be a file path (str or Path) or a file-like object.
-        :param visibility: to set the visibility of the uploaded replay.
+        :param visibility: to set the visibility of the uploaded replay ('public', 'unlisted', 'private').
         :param group: to upload the replay to an existing group.
-        :return: the result of the POST request.
+        :param raise_on_duplicate: if True, raise an HTTPError when a 409 Conflict (duplicate replay)
+                                   is returned. If False (default), returns the API response dict with the
+                                   existing replay id.
+        :return: the result of the POST request (dict containing replay 'id').
         """
         if isinstance(replay_file, (str, Path)):
             with open(replay_file, "rb") as f:
-                return self.upload_replay(f, visibility=visibility, group=group)
-        return self._request(f"/v2/upload", "POST", files={"file": replay_file},
-                             params={"group": group, "visibility": visibility}).json()
-
+                return self.upload_replay(
+                    f,
+                    visibility=visibility,
+                    group=group,
+                    raise_on_duplicate=raise_on_duplicate,
+                )
+        r = self._request(
+            "/v2/upload",
+            "POST",
+            files={"file": replay_file},
+            params={"group": group, "visibility": visibility},
+            allow_status_codes=(409,),
+            category=RateLimitCategory.CORE,
+        )
+        if r.status_code == 409 and raise_on_duplicate:
+            r.raise_for_status()
+        return r.json()
+
+    @rate_limited(RateLimitCategory.CORE)
     def delete_replay(self, replay_id: str) -> None:
         """
         This endpoint deletes the specified replay.
@@ -355,8 +432,9 @@ class BallchasingApi:
 
         :param replay_id: the replay id.
         """
-        self._request(f"/replays/{replay_id}", "DELETE")
+        self._request(f"/replays/{replay_id}", "DELETE", category=RateLimitCategory.CORE)
 
+    @rate_limited(RateLimitCategory.LIST)
     def get_groups(
             self,
             *,
@@ -385,14 +463,14 @@ class BallchasingApi:
                               RFC3339 format, e.g. 2020-01-02T15:00:05+01:00
         :param count: returns at most count groups. Since the implementation uses an iterator it supports iterating
                       past the limit of 200 set by the API
-        :param sort_by: Sort groups according the selected field.
-        :param sort_dir: Sort direction.
+        :param sort_by: Sort groups according the selected field ('created', 'name').
+        :param sort_dir: Sort direction ('asc', 'desc').
         :param deep: whether to get full stats for each group (will be much slower).
         :param typed: whether to return a typed object (default is self.typed).
         :param disable_prefetch: whether to disable prefetching.
         :return: an iterator over the groups returned by the API.
         """
-        url = f"{self.base_url}/groups/"
+        url = f"{self.base_url}/groups"
         params = {"name": name, "creator": creator, "group": group, "created-before": to_rfc3339(created_before),
                   "created-after": to_rfc3339(created_after), "count": count, "sort-by": sort_by, "sort-dir": sort_dir}
 
@@ -408,6 +486,7 @@ class BallchasingApi:
             iterator = (ShallowGroup(**g) for g in iterator)
         yield from iterator
 
+    @rate_limited(RateLimitCategory.CORE)
     def create_group(
             self,
             *,
@@ -420,7 +499,7 @@ class BallchasingApi:
         Use this API to create a new replay group.
 
         :param name: the new group name.
-        :param player_identification: how to identify the same player across multiple replays.
+        :param player_identification: how to identify the same player across multiple replays ('by-id', 'by-name').
                                       Some tournaments (e.g. RLCS) make players use a pool of generic Steam accounts,
                                       meaning the same player could end up using 2 different accounts in 2 series.
                                       That's when the `by-name` comes in handy
@@ -428,13 +507,19 @@ class BallchasingApi:
                                     Set to `by-distinct-players` if teams have a fixed roster of players for
                                     every single game. In some tournaments/leagues, teams allow player rotations,
                                     or a sub can replace another player, in which case use `by-player-clusters`.
-        :param parent: if set,the new group will be created as a child of the specified group
+        :param parent: if set, the new group will be created as a child of the specified group
         :return: the result of the POST request.
         """
-        json = {"name": name, "player_identification": player_identification,
-                "team_identification": team_identification, "parent": parent}
-        return self._request(f"/groups", "POST", json=json).json()
+        json = {
+            "name": name,
+            "player_identification": player_identification,
+            "team_identification": team_identification,
+        }
+        if parent is not None:
+            json["parent"] = parent
+        return self._request(f"/groups", "POST", json=json, category=RateLimitCategory.CORE).json()
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_group(
             self,
             group_id: str,
@@ -448,22 +533,46 @@ class BallchasingApi:
         :param typed: whether to return a typed object (default is self.typed).
         :return: the group info with stats.
         """
-        result = self._request(f"/groups/{group_id}", "GET").json()
+        result = self._request(f"/groups/{group_id}", "GET", category=RateLimitCategory.CORE).json()
         if typed is None:
             typed = self.typed
         if typed:
             result = DeepGroup(**result)
         return result
 
-    def patch_group(self, group_id: str, **params) -> None:
+    @rate_limited(RateLimitCategory.CORE)
+    def patch_group(
+            self,
+            group_id: str,
+            *,
+            player_identification: AnyPlayerIdentification | None = None,
+            team_identification: AnyTeamIdentification | None = None,
+            parent: str | None = None,
+            shared: bool | None = None,
+            **params
+    ) -> None:
         """
         This endpoint can patch one or more fields of the specified group.
 
         :param group_id: the group id
-        :param params: parameters for the PATCH request.
-        """
-        self._request(f"/groups/{group_id}", "PATCH", json=params)
-
+        :param player_identification: how to identify the same player across multiple replays ('by-id', 'by-name').
+        :param team_identification: how to identify the same team ('by-distinct-players', 'by-player-clusters').
+        :param parent: parent group id to assign, or "" to unparent.
+        :param shared: whether to mark the group as shared.
+        :param params: additional parameters for the PATCH request.
+        """
+        payload = dict(params)
+        if player_identification is not None:
+            payload["player_identification"] = player_identification
+        if team_identification is not None:
+            payload["team_identification"] = team_identification
+        if parent is not None:
+            payload["parent"] = parent
+        if shared is not None:
+            payload["shared"] = shared
+        self._request(f"/groups/{group_id}", "PATCH", json=payload, category=RateLimitCategory.CORE)
+
+    @rate_limited(RateLimitCategory.CORE)
     def delete_group(self, group_id: str) -> None:
         """
         This endpoint deletes the specified group.
@@ -471,7 +580,7 @@ class BallchasingApi:
 
         :param group_id: the group id.
         """
-        self._request(f"/groups/{group_id}", "DELETE")
+        self._request(f"/groups/{group_id}", "DELETE", category=RateLimitCategory.CORE)
 
     def get_group_replays(
             self,
@@ -519,22 +628,25 @@ class BallchasingApi:
         for replay in self.get_replays(group_id=group_id, deep=deep, typed=typed):
             yield [group_id], replay
 
-    def download_replay(self, replay_id: str, path: str):
+    @rate_limited(RateLimitCategory.DOWNLOAD)
+    def download_replay(self, replay_id: str, path: str | Path) -> Path:
         """
         Download a replay file.
 
         :param replay_id: the replay id.
         :param path: the path to download the replay to. Can be a file path or a directory.
+        :return: the Path to the downloaded file.
         """
-        r = self._request(f"/replays/{replay_id}/file", "GET")
-        if os.path.isdir(path):
-            # If path is a directory, use the replay id as the filename
-            filename = f"{replay_id}.replay"
-            path = os.path.join(path, filename)
-        with open(path, "wb") as f:
-            f.write(r.content)
+        dest = Path(path)
+        if dest.is_dir():
+            dest = dest / f"{replay_id}.replay"
+        r = self._request(f"/replays/{replay_id}/file", "GET", stream=True, category=RateLimitCategory.DOWNLOAD)
+        with open(dest, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+        return dest
 
-    def download_group(self, group_id: str, folder: str, *, keep_tree_structure=True):
+    def download_group(self, group_id: str, folder: str | Path, *, keep_tree_structure=True):
         """
         Download an entire group.
 
@@ -542,22 +654,24 @@ class BallchasingApi:
         :param folder: the folder in which to create the group folder.
         :param keep_tree_structure: whether to create new folders for child groups.
         """
-        folder = os.path.join(folder, group_id)
+        folder = Path(folder) / group_id
         if keep_tree_structure:
-            os.makedirs(folder, exist_ok=True)
+            folder.mkdir(parents=True, exist_ok=True)
             for child_group in self.get_groups(group=group_id):
                 self.download_group(child_group["id"], folder, keep_tree_structure=True)
             for replay in self.get_replays(group_id=group_id):
                 self.download_replay(replay["id"], folder)
         else:
+            folder.mkdir(parents=True, exist_ok=True)
             for replay in self.get_group_replays(group_id):
                 self.download_replay(replay["id"], folder)
 
+    @rate_limited(RateLimitCategory.CORE)
     def get_maps(self):
         """
         Use this API to get the list of map codes to map names (map as in stadium).
         """
-        res = self._request("/maps", "GET").json()
+        res = self._request("/maps", "GET", category=RateLimitCategory.CORE).json()
         return res
 
     def get_stats(self, replay: dict | str):

python_ballchasing-0.5.3/ballchasing/rate_limit.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import functools
+import threading
+import time
+from enum import Enum
+from typing import Callable, Any
+
+
+class RateLimitCategory(str, Enum):
+    """Rate limit categories defined in ballchasing.com API documentation."""
+    LIST = "list"          # GET /replays, GET /groups
+    CORE = "core"          # GET/PATCH/DELETE /replays/{id}, /groups/{id}, POST /groups, upload, maps
+    DOWNLOAD = "download"  # GET /replays/{id}/file
+    PING = "ping"          # GET /
+
+
+# Hourly rate limit backoff times per category (seconds to wait between requests)
+# - regular: list 500/hr (7.2s), core 1000/hr (3.6s), download 200/hr (18.0s)
+# - gold: list 1000/hr (3.6s), core 2000/hr (1.8s), download 400/hr (9.0s)
+# - diamond: list 2000/hr (1.8s), core 5000/hr (0.72s), download 1000/hr (3.6s)
+# - champion: list 8/s, core 8/s, download 2000/hr (1.8s)
+# - gc: list 16/s, core 16/s, download 2/s (0.5s)
+CATEGORY_HOURLY_SLEEP_TIMES: dict[str, dict[RateLimitCategory, float]] = {
+    "regular": {
+        RateLimitCategory.LIST: 3600 / 500,      # 7.2s
+        RateLimitCategory.CORE: 3600 / 1000,     # 3.6s
+        RateLimitCategory.DOWNLOAD: 3600 / 200,  # 18.0s
+        RateLimitCategory.PING: 0.0,
+    },
+    "gold": {
+        RateLimitCategory.LIST: 3600 / 1000,     # 3.6s
+        RateLimitCategory.CORE: 3600 / 2000,     # 1.8s
+        RateLimitCategory.DOWNLOAD: 3600 / 400,  # 9.0s
+        RateLimitCategory.PING: 0.0,
+    },
+    "diamond": {
+        RateLimitCategory.LIST: 3600 / 2000,     # 1.8s
+        RateLimitCategory.CORE: 3600 / 5000,     # 0.72s
+        RateLimitCategory.DOWNLOAD: 3600 / 1000, # 3.6s
+        RateLimitCategory.PING: 0.0,
+    },
+    "champion": {
+        RateLimitCategory.LIST: 1.0 / 8,         # 0.125s
+        RateLimitCategory.CORE: 1.0 / 8,         # 0.125s
+        RateLimitCategory.DOWNLOAD: 3600 / 2000, # 1.8s
+        RateLimitCategory.PING: 0.0,
+    },
+    "gc": {
+        RateLimitCategory.LIST: 1.0 / 16,        # 0.0625s
+        RateLimitCategory.CORE: 1.0 / 16,        # 0.0625s
+        RateLimitCategory.DOWNLOAD: 1.0 / 2,     # 0.5s
+        RateLimitCategory.PING: 0.0,
+    },
+}
+
+# Maximum calls per second (burst rates)
+CATEGORY_BURST_RATES: dict[str, dict[RateLimitCategory, float]] = {
+    "regular": {RateLimitCategory.LIST: 2.0, RateLimitCategory.CORE: 2.0, RateLimitCategory.DOWNLOAD: 1.0, RateLimitCategory.PING: 10.0},
+    "gold": {RateLimitCategory.LIST: 2.0, RateLimitCategory.CORE: 2.0, RateLimitCategory.DOWNLOAD: 2.0, RateLimitCategory.PING: 10.0},
+    "diamond": {RateLimitCategory.LIST: 4.0, RateLimitCategory.CORE: 4.0, RateLimitCategory.DOWNLOAD: 2.0, RateLimitCategory.PING: 10.0},
+    "champion": {RateLimitCategory.LIST: 8.0, RateLimitCategory.CORE: 8.0, RateLimitCategory.DOWNLOAD: 2.0, RateLimitCategory.PING: 10.0},
+    "gc": {RateLimitCategory.LIST: 16.0, RateLimitCategory.CORE: 16.0, RateLimitCategory.DOWNLOAD: 2.0, RateLimitCategory.PING: 10.0},
+}
+
+
+def resolve_category(url_or_endpoint: str) -> RateLimitCategory:
+    """Categorize an endpoint URL or path into a RateLimitCategory."""
+    path = url_or_endpoint.split("?")[0].rstrip("/")
+    if path.endswith("/file"):
+        return RateLimitCategory.DOWNLOAD
+    if path.endswith("/replays") or path.endswith("/groups"):
+        return RateLimitCategory.LIST
+    if path == "" or path.endswith("/api") or path.endswith("/api/"):
+        return RateLimitCategory.PING
+    return RateLimitCategory.CORE
+
+
+class RateLimiter:
+    """
+    Thread-safe rate limiter and global 429 tracker for BallchasingApi.
+
+    Coordinates rate limits across concurrent requests/threads sharing the same
+    BallchasingApi instance to prevent cascading 429 errors.
+    """
+
+    def __init__(
+            self,
+            patron_type_getter: Callable[[], str | None] | None = None,
+            *,
+            proactive: bool = False,
+    ):
+        self._lock = threading.Lock()
+        self._patron_type_getter = patron_type_getter
+        self.proactive = proactive
+
+        # Global reactive cooldown (shared across all threads/endpoints for this API instance)
+        self._rate_limited_until: float = 0.0
+        self._last_rate_limited_at: float | None = None
+
+        # Per-category tracking
+        self._last_request_time: dict[RateLimitCategory, float] = {}
+        self._category_rate_limit_count: dict[RateLimitCategory, int] = {}
+        self._last_rate_limited_per_category: dict[RateLimitCategory, float] = {}
+
+    @property
+    def last_rate_limited_at(self) -> float | None:
+        """Timestamp (time.time()) of the most recent 429 response, or None if never rate-limited."""
+        with self._lock:
+            return self._last_rate_limited_at
+
+    @property
+    def is_rate_limited(self) -> bool:
+        """True if the instance is currently waiting out a 429 rate-limit cooldown."""
+        with self._lock:
+            return time.monotonic() < self._rate_limited_until
+
+    @property
+    def rate_limited_until(self) -> float:
+        """Monotonic timestamp until which the global cooldown is active."""
+        with self._lock:
+            return self._rate_limited_until
+
+    def get_patron_type(self) -> str:
+        if self._patron_type_getter:
+            try:
+                pt = self._patron_type_getter()
+                if pt:
+                    return pt
+            except Exception:
+                pass
+        return "regular"
+
+    def get_category_sleep_time(self, category: RateLimitCategory) -> float:
+        """Returns the hourly backoff sleep time for the specified category and current patron tier."""
+        pt = self.get_patron_type()
+        return CATEGORY_HOURLY_SLEEP_TIMES.get(pt, CATEGORY_HOURLY_SLEEP_TIMES["regular"]).get(
+            category, 3.6
+        )
+
+    def before_request(self, category: RateLimitCategory) -> None:
+        """
+        Called before sending an HTTP request.
+        If a global 429 cooldown is active, waits for it to expire.
+        If proactive rate limiting is enabled, paces requests according to the category's burst rate.
+        """
+        sleep_duration = 0.0
+        with self._lock:
+            now_mono = time.monotonic()
+            # 1. Check global reactive 429 cooldown
+            if now_mono < self._rate_limited_until:
+                sleep_duration = self._rate_limited_until - now_mono
+
+            # 2. If proactive mode, ensure spacing between requests
+            if self.proactive:
+                pt = self.get_patron_type()
+                rate = CATEGORY_BURST_RATES.get(pt, CATEGORY_BURST_RATES["regular"]).get(category, 2.0)
+                min_interval = 1.0 / rate if rate > 0 else 0.0
+                last_time = self._last_request_time.get(category, 0.0)
+                elapsed = now_mono - last_time
+                if elapsed < min_interval:
+                    proactive_wait = min_interval - elapsed
+                    sleep_duration = max(sleep_duration, proactive_wait)
+
+                self._last_request_time[category] = now_mono + sleep_duration
+
+        if sleep_duration > 0:
+            time.sleep(sleep_duration)
+
+    def record_429(
+            self,
+            category: RateLimitCategory,
+            *,
+            retry_after: float | None = None,
+            fallback_sleep_time: float | None = None,
+            retries: int = 1,
+    ) -> float:
+        """
+        Record a 429 response. Updates the global and per-category tracker,
+        and returns the duration (in seconds) that the caller should sleep.
+        """
+        with self._lock:
+            now_wall = time.time()
+            now_mono = time.monotonic()
+
+            self._last_rate_limited_at = now_wall
+            self._last_rate_limited_per_category[category] = now_wall
+            self._category_rate_limit_count[category] = self._category_rate_limit_count.get(category, 0) + 1
+
+            if retry_after is not None and retry_after > 0:
+                wait_time = float(retry_after)
+            elif fallback_sleep_time is not None and fallback_sleep_time > 0:
+                wait_time = float(fallback_sleep_time)
+            else:
+                cat_sleep = self.get_category_sleep_time(category)
+                backoff = float(2 ** (retries - 1))
+                wait_time = max(cat_sleep, min(backoff, 60.0))
+
+            # Extend global cooldown until this wait expires
+            self._rate_limited_until = max(self._rate_limited_until, now_mono + wait_time)
+            return wait_time
+
+    def get_stats(self) -> dict[str, Any]:
+        """Returns statistics on rate limits encountered per category."""
+        with self._lock:
+            return {
+                "last_rate_limited_at": self._last_rate_limited_at,
+                "is_rate_limited": time.monotonic() < self._rate_limited_until,
+                "category_counts": {k.value: v for k, v in self._category_rate_limit_count.items()},
+                "last_per_category": {k.value: v for k, v in self._last_rate_limited_per_category.items()},
+            }
+
+
+def rate_limited(category: RateLimitCategory | str | None = None):
+    """
+    Decorator for BallchasingApi methods to declare their RateLimitCategory
+    and ensure proper categorization.
+    """
+    cat = RateLimitCategory(category) if category else RateLimitCategory.CORE
+
+    def decorator(fn):
+        @functools.wraps(fn)
+        def wrapper(self, *args, **kwargs):
+            return fn(self, *args, **kwargs)
+        wrapper.rate_limit_category = cat
+        return wrapper
+    return decorator

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python-ballchasing"
-version = "0.5.2"
+version = "0.5.3"
 authors = [
     { name = "Rolv-Arild Braaten", email = "rolv_arild@hotmail.com" },
 ]

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/python_ballchasing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-ballchasing
-Version: 0.5.2
+Version: 0.5.3
 Summary: A Python wrapper around the Ballchasing API
 Author-email: Rolv-Arild Braaten <rolv_arild@hotmail.com>
 License: MIT License

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/python_ballchasing.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 ballchasing/__init__.py
 ballchasing/api.py
 ballchasing/constants.py
+ballchasing/rate_limit.py
 ballchasing/stats_info.tsv
 ballchasing/typed/__init__.py
 ballchasing/typed/deep_group.py

{python_ballchasing-0.5.2 → python_ballchasing-0.5.3}/tests/test_fixes.py

@@ -1,25 +1,33 @@
+import io
 import logging
+import time
+from pathlib import Path
 from unittest.mock import patch, MagicMock
 import pytest
 from requests import Response
-from requests.exceptions import ConnectionError as RequestsConnectionError, Timeout
+from requests.exceptions import ConnectionError as RequestsConnectionError, HTTPError, Timeout
 
 from ballchasing.api import BallchasingApi, PATRON_RATE_LIMIT_SLEEP_TIMES
 from ballchasing.constants import Map
+from ballchasing.rate_limit import RateLimiter, RateLimitCategory, resolve_category, rate_limited
 from ballchasing.typed.shared import PlayerID, BasePlayer, BaseReplay
 from ballchasing.typed.deep_replay import DeepReplay, TeamCoreStatsDR, PlayerCoreStatsDR
 from ballchasing.util.replays import is_standard_replay
 
 
-def make_mock_response(status_code=200, json_data=None, headers=None):
+def make_mock_response(status_code=200, json_data=None, headers=None, content=None):
     resp = Response()
     resp.status_code = status_code
-    if json_data is not None:
-        resp._content = (
-            bytes(__import__("json").dumps(json_data), "utf-8")
-        )
+    if content is not None:
+        resp._content = content
+        resp.raw = io.BytesIO(content)
+    elif json_data is not None:
+        raw_bytes = bytes(__import__("json").dumps(json_data), "utf-8")
+        resp._content = raw_bytes
+        resp.raw = io.BytesIO(raw_bytes)
     else:
         resp._content = b"{}"
+        resp.raw = io.BytesIO(b"{}")
     if headers:
         resp.headers.update(headers)
     return resp
@@ -94,7 +102,7 @@ class TestHttpRetryAndLogging:
              patch("time.sleep") as mock_sleep:
             res = api._request("/test", "GET")
             assert res.status_code == 200
-            mock_sleep.assert_called_once_with(3)
+            mock_sleep.assert_called_once_with(3.0)
 
     def test_rate_limit_backoff_default_when_unset(self):
         api = BallchasingApi("dummy_key", sleep_time_on_rate_limit=0, do_initial_ping=False)
@@ -108,6 +116,18 @@ class TestHttpRetryAndLogging:
             assert mock_sleep.call_count == 1
             assert mock_sleep.call_args[0][0] >= 1.0
 
+    def test_rate_limit_retries_indefinitely(self):
+        api = BallchasingApi("dummy_key", sleep_time_on_rate_limit=0.125, do_initial_ping=False)
+        # Simulate 10 consecutive 429s followed by a 200 OK
+        rl_resps = [make_mock_response(status_code=429) for _ in range(10)]
+        ok_resp = make_mock_response(status_code=200, json_data={"ok": True})
+
+        with patch.object(api._session, "request", side_effect=rl_resps + [ok_resp]), \
+             patch("time.sleep") as mock_sleep:
+            res = api._request("/test", "GET")
+            assert res.status_code == 200
+            assert mock_sleep.call_count == 10
+
     def test_logging_used_on_retry(self, caplog):
         api = BallchasingApi("dummy_key", do_initial_ping=False, print_on_rate_limit=True)
         rl_resp = make_mock_response(status_code=429)
@@ -244,3 +264,167 @@ class TestNonePlayerIdFixes:
         valid_nobots, reason_nobots = is_standard_replay(replay, allow_bots=False)
         assert valid_nobots is False
         assert "Bot1 has no ID" in reason_nobots
+
+
+class TestApiDocReflections:
+    def test_upload_replay_returns_id_on_duplicate_409_when_not_raising(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        resp_409 = make_mock_response(409, json_data={"id": "existing-replay-id", "error": "duplicate replay"})
+
+        with patch.object(api._session, "request", return_value=resp_409):
+            result = api.upload_replay(io.BytesIO(b"dummy_replay_data"), raise_on_duplicate=False)
+            assert result["id"] == "existing-replay-id"
+
+    def test_upload_replay_raises_on_duplicate_409_when_configured(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        resp_409 = make_mock_response(409, json_data={"id": "existing-replay-id", "error": "duplicate replay"})
+
+        with patch.object(api._session, "request", return_value=resp_409):
+            with pytest.raises(HTTPError):
+                api.upload_replay(io.BytesIO(b"dummy_replay_data"), raise_on_duplicate=True)
+
+    def test_create_group_omits_parent_when_none(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(201, json_data={"id": "grp1", "name": "Top Level"})
+
+        with patch.object(api._session, "request", return_value=ok_resp) as mock_req:
+            res = api.create_group(name="Top Level", player_identification="by-id", team_identification="by-distinct-players")
+            assert res["id"] == "grp1"
+            assert mock_req.call_count == 1
+            payload = mock_req.call_args[1]["json"]
+            assert "parent" not in payload
+            assert payload == {
+                "name": "Top Level",
+                "player_identification": "by-id",
+                "team_identification": "by-distinct-players"
+            }
+
+    def test_create_group_includes_parent_when_provided(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(201, json_data={"id": "grp2", "name": "Child Group"})
+
+        with patch.object(api._session, "request", return_value=ok_resp) as mock_req:
+            api.create_group(name="Child Group", player_identification="by-id", team_identification="by-distinct-players", parent="parent-grp")
+            payload = mock_req.call_args[1]["json"]
+            assert payload["parent"] == "parent-grp"
+
+    def test_patch_replay_explicit_kwargs(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(204)
+
+        with patch.object(api._session, "request", return_value=ok_resp) as mock_req:
+            api.patch_replay("rep-123", title="New Title", visibility="private", group="")
+            payload = mock_req.call_args[1]["json"]
+            assert payload == {"title": "New Title", "visibility": "private", "group": ""}
+
+    def test_patch_group_explicit_kwargs(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(204)
+
+        with patch.object(api._session, "request", return_value=ok_resp) as mock_req:
+            api.patch_group("grp-123", player_identification="by-name", team_identification="by-player-clusters", parent="new-parent", shared=True)
+            payload = mock_req.call_args[1]["json"]
+            assert payload == {
+                "player_identification": "by-name",
+                "team_identification": "by-player-clusters",
+                "parent": "new-parent",
+                "shared": True
+            }
+
+    def test_get_groups_url_no_trailing_slash(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(200, json_data={"list": []})
+
+        with patch.object(api._session, "request", return_value=ok_resp) as mock_req:
+            list(api.get_groups(count=10, disable_prefetch=True))
+            called_url = mock_req.call_args[1]["url"]
+            assert called_url == f"{api.base_url}/groups"
+
+    def test_download_replay_supports_pathlib(self, tmp_path):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        ok_resp = make_mock_response(200, content=b"PK\x03\x04fake_replay_binary")
+
+        with patch.object(api._session, "request", return_value=ok_resp):
+            dest = api.download_replay("rep-123", tmp_path)
+            assert dest == tmp_path / "rep-123.replay"
+            assert dest.read_bytes() == b"PK\x03\x04fake_replay_binary"
+
+
+class TestRateLimitingAndTracker:
+    def test_initial_state_not_rate_limited(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        assert api.is_rate_limited is False
+        assert api.last_rate_limited_at is None
+        stats = api.rate_limit_stats
+        assert stats["is_rate_limited"] is False
+        assert stats["last_rate_limited_at"] is None
+        assert stats["category_counts"] == {}
+
+    def test_category_resolution(self):
+        assert resolve_category("/replays") == RateLimitCategory.LIST
+        assert resolve_category("https://ballchasing.com/api/groups") == RateLimitCategory.LIST
+        assert resolve_category("/replays/123/file") == RateLimitCategory.DOWNLOAD
+        assert resolve_category("/replays/abc-123") == RateLimitCategory.CORE
+        assert resolve_category("/groups/grp-456") == RateLimitCategory.CORE
+        assert resolve_category("/v2/upload") == RateLimitCategory.CORE
+        assert resolve_category("/maps") == RateLimitCategory.CORE
+        assert resolve_category("/") == RateLimitCategory.PING
+
+    def test_record_429_updates_tracker_and_stats(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        rl_resp = make_mock_response(status_code=429, headers={"Retry-After": "5"})
+        ok_resp = make_mock_response(status_code=200, json_data={"list": []})
+
+        with patch.object(api._session, "request", side_effect=[rl_resp, ok_resp]), \
+             patch("time.sleep"):
+            api._request("/replays", "GET")
+
+        assert api.last_rate_limited_at is not None
+        assert time.time() - api.last_rate_limited_at < 5.0
+        stats = api.rate_limit_stats
+        assert stats["category_counts"].get("list") == 1
+        assert "list" in stats["last_per_category"]
+
+    def test_category_sleep_times(self):
+        limiter = RateLimiter(patron_type_getter=lambda: "regular")
+        assert limiter.get_category_sleep_time(RateLimitCategory.LIST) == 3600 / 500  # 7.2s
+        assert limiter.get_category_sleep_time(RateLimitCategory.CORE) == 3600 / 1000  # 3.6s
+        assert limiter.get_category_sleep_time(RateLimitCategory.DOWNLOAD) == 3600 / 200  # 18.0s
+
+        gold_limiter = RateLimiter(patron_type_getter=lambda: "gold")
+        assert gold_limiter.get_category_sleep_time(RateLimitCategory.LIST) == 3600 / 1000  # 3.6s
+        assert gold_limiter.get_category_sleep_time(RateLimitCategory.CORE) == 3600 / 2000  # 1.8s
+        assert gold_limiter.get_category_sleep_time(RateLimitCategory.DOWNLOAD) == 3600 / 400  # 9.0s
+
+    def test_global_tracker_blocks_concurrent_requests(self):
+        # Simulate instance being put into cooldown
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        api.rate_limiter.record_429(RateLimitCategory.CORE, retry_after=5.0)
+
+        assert api.is_rate_limited is True
+        # A subsequent call to before_request should sleep for the remaining cooldown
+        with patch("time.sleep") as mock_sleep:
+            api.rate_limiter.before_request(RateLimitCategory.LIST)
+            assert mock_sleep.call_count == 1
+            slept = mock_sleep.call_args[0][0]
+            assert 0.0 < slept <= 5.0
+
+    def test_proactive_mode_enforces_burst_rate(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False, proactive_rate_limit=True)
+        # On proactive mode, 2 calls in immediate succession will sleep for min_interval
+        with patch("time.sleep") as mock_sleep:
+            api.rate_limiter.before_request(RateLimitCategory.LIST)
+            # First call has no elapsed constraint
+            assert mock_sleep.call_count == 0
+            # Immediate second call should sleep
+            api.rate_limiter.before_request(RateLimitCategory.LIST)
+            assert mock_sleep.call_count == 1
+            slept = mock_sleep.call_args[0][0]
+            assert 0.4 <= slept <= 0.6  # 1 / 2.0s = 0.5s interval
+
+    def test_rate_limited_decorator_attaches_to_api_methods(self):
+        api = BallchasingApi("dummy_key", do_initial_ping=False)
+        assert getattr(api.get_replay, "rate_limit_category", None) == RateLimitCategory.CORE
+        assert getattr(api.get_replays, "rate_limit_category", None) == RateLimitCategory.LIST
+        assert getattr(api.download_replay, "rate_limit_category", None) == RateLimitCategory.DOWNLOAD
+        assert getattr(api.ping, "rate_limit_category", None) == RateLimitCategory.PING