shindan-cli 2.2.1__tar.gz → 2.2.2__tar.gz

{shindan_cli-2.2.1 → shindan_cli-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: shindan-cli
-Version: 2.2.1
+Version: 2.2.2
 Summary: ShindanMaker (https://shindanmaker.com) CLI
 Project-URL: Repository, https://github.com/eggplants/shindan-cli
 Author-email: eggplants <w10776e8w@yahoo.co.jp>
@@ -57,7 +57,7 @@ Supported types of diagnosis:
 
 - [Name-based diagnosis (名前診断)](https://shindanmaker.com/list/name)
 - [Branching diagnosis (分岐診断)](https://shindanmaker.com/list/branch)
-- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai)
+- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai) (currently blocked by a Cloudflare Turnstile challenge)
 - [Check Diagnosis (チェック診断)](https://shindanmaker.com/list/check)
 
 ### CLI

{shindan_cli-2.2.1 → shindan_cli-2.2.2}/README.md

@@ -32,7 +32,7 @@ Supported types of diagnosis:
 
 - [Name-based diagnosis (名前診断)](https://shindanmaker.com/list/name)
 - [Branching diagnosis (分岐診断)](https://shindanmaker.com/list/branch)
-- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai)
+- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai) (currently blocked by a Cloudflare Turnstile challenge)
 - [Check Diagnosis (チェック診断)](https://shindanmaker.com/list/check)
 
 ### CLI

shindan_cli-2.2.2/shindan_cli/_http.py

@@ -0,0 +1,79 @@
+"""HTTP helpers to cope with rate limiting on <https://shindanmaker.com>."""
+
+from __future__ import annotations
+
+import random
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from requests import Response, Session
+
+TOO_MANY_REQUESTS = 429
+
+MAX_RETRIES = 5
+INITIAL_WAIT = 2.0
+BACKOFF_FACTOR = 2.0
+
+WAIT_RANGE = (2.0, 5.0)
+
+
+def random_wait() -> None:
+    """Sleep for a random while, to fetch at a less machine-like pace."""
+    time.sleep(random.uniform(*WAIT_RANGE))  # noqa: S311
+
+
+def backoff_wait(attempt: int) -> float:
+    """Get how long to wait before the given retry attempt.
+
+    Args:
+        attempt (int): 0-based index of the retry about to be made
+
+    Returns:
+        float: seconds to sleep
+
+    """
+    return INITIAL_WAIT * BACKOFF_FACTOR**attempt
+
+
+def request_with_retry(
+    session: Session,
+    method: str,
+    url: str,
+    **kwargs: Any,  # noqa: ANN401
+) -> Response:
+    """Send a request, retrying with a backoff while the site rate limits us.
+
+    The site is fronted by Cloudflare, which answers bursts of requests with a
+    `429` challenge page instead of the result. Those responses are transient,
+    so they are retried rather than reported to the caller.
+
+    Args:
+        session (Session): session object
+        method (str): HTTP method
+        url (str): url to request
+        **kwargs (Any): extra arguments passed to `Session.request`
+
+    Returns:
+        Response: the last response received
+
+    """
+    response = session.request(method, url, **kwargs)
+    for attempt in range(MAX_RETRIES - 1):
+        if response.status_code != TOO_MANY_REQUESTS:
+            break
+        time.sleep(backoff_wait(attempt))
+        response = session.request(method, url, **kwargs)
+    return response
+
+
+__all__ = (
+    "BACKOFF_FACTOR",
+    "INITIAL_WAIT",
+    "MAX_RETRIES",
+    "TOO_MANY_REQUESTS",
+    "WAIT_RANGE",
+    "backoff_wait",
+    "random_wait",
+    "request_with_retry",
+)

{shindan_cli-2.2.1 → shindan_cli-2.2.2}/shindan_cli/get_results.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, cast
 
 from bs4 import BeautifulSoup, Tag
 
+from ._http import MAX_RETRIES, backoff_wait, request_with_retry
 from .constants import HEADERS, AIParams, BranchParams, CheckParams, NameParams
 
 if TYPE_CHECKING:
@@ -17,9 +18,6 @@ if TYPE_CHECKING:
 
 Params = AIParams | BranchParams | CheckParams | NameParams
 
-_MAX_RETRIES = 3
-_RETRY_WAIT = 4.0
-
 
 def __get_result(
     session: Session,
@@ -29,15 +27,19 @@ def __get_result(
     shindan_url: str,
 ) -> ShindanResult:
     result_tag: Tag | None = None
+    status_code = None
 
-    for attempt in range(_MAX_RETRIES):
+    for attempt in range(MAX_RETRIES):
         if attempt > 0:
-            time.sleep(_RETRY_WAIT)
-        result_page = session.post(
+            time.sleep(backoff_wait(attempt - 1))
+        result_page = request_with_retry(
+            session,
+            "POST",
             shindan_url + ("/r" if is_renewal else ""),
             data=params,
             headers=HEADERS,
         )
+        status_code = result_page.status_code
         soup = BeautifulSoup(result_page.text, features="lxml")
         found = soup.find(id="share-copytext-shindanresult-textarea")
         if isinstance(found, Tag) and found.text:
@@ -45,7 +47,7 @@ def __get_result(
             break
 
     if result_tag is None:
-        msg = f"Could not find a tag contains the result, returns: {result_tag}"
+        msg = f"Could not find a tag contains the result, last status code: {status_code}"
         raise TypeError(msg)
 
     *results, hashtag, shindan_url, _ = result_tag.text.split("\n")
@@ -85,7 +87,9 @@ def get_result_by_ai(
     """
     ai_headers = {**HEADERS, "x-csrf-token": csrf_token}
 
-    result_sse = session.post(
+    result_sse = request_with_retry(
+        session,
+        "POST",
         f"{shindan_url}/ai_result",
         json={
             "form_values": user_inputs,

{shindan_cli-2.2.1 → shindan_cli-2.2.2}/shindan_cli/shindan.py

@@ -2,14 +2,14 @@
 
 from __future__ import annotations
 
-import random
-import time
 from typing import cast
 
 import cloudscraper  # type: ignore[unused-ignore,import-not-found,import-untyped]
 from bs4 import BeautifulSoup
 from requests.sessions import Session
 
+from . import _http
+from ._http import request_with_retry
 from .constants import BASE_URL, HEADERS, TARGET_KEYS_BY_TYPE
 from .get_results import (
     Params,
@@ -26,6 +26,42 @@ class ShindanError(Exception):
     """Error class for shindan-cli."""
 
 
+def __get_csrf_token(session: Session, source: BeautifulSoup) -> str:
+    """Get a CSRF token to submit the shindan form with.
+
+    The shindan pages are served from a cache with their token fields left
+    blank, and the browser fills them in from `/csrf-token` just before
+    submitting the form.
+
+    Args:
+        session (Session): session object
+        source (BeautifulSoup): parsed shindan page
+
+    Returns:
+        str: CSRF token
+
+    Raises:
+        ShindanError
+
+    """
+    csrf_meta = source.select_one('meta[name="csrf-token"]')
+    if csrf_meta and isinstance(token := csrf_meta.get("content"), str) and token:
+        return token
+
+    response = request_with_retry(
+        session,
+        "GET",
+        f"{BASE_URL}/csrf-token",
+        headers={**HEADERS, "X-Requested-With": "XMLHttpRequest"},
+    )
+    if response.status_code != 200:  # noqa: PLR2004
+        raise ShindanError(response.status_code)
+    if not isinstance(token := response.json().get("token"), str) or not token:
+        msg = "Could not find CSRF token on the shindan page."
+        raise ShindanError(msg)
+    return token
+
+
 def shindan(
     page_id: int,
     shindan_name: str,
@@ -57,7 +93,7 @@ def shindan(
     session = cloudscraper.create_scraper()
     assert isinstance(session, Session)  # noqa: S101
 
-    shindan_page = session.get(shindan_url, headers=HEADERS)
+    shindan_page = request_with_retry(session, "GET", shindan_url, headers=HEADERS)
     if shindan_page.status_code != 200:  # noqa: PLR2004
         raise ShindanError(shindan_page.status_code)
 
@@ -78,9 +114,10 @@ def shindan(
     )
     # overwrite randname (old: shindanName)
     params["randname"] = shindan_name
+    params["_token"] = params["_token"] or __get_csrf_token(session, source)
 
     if wait:
-        time.sleep(random.uniform(2, 5))  # noqa: S311
+        _http.random_wait()
 
     if params["type"] == "ai":
         hashtag_title = source.select_one("h1#shindanTitle")
@@ -89,16 +126,12 @@ def shindan(
             str,
         ):
             hashtag = None
-        csrf_meta = source.select_one('meta[name="csrf-token"]')
-        if not csrf_meta or not isinstance(csrf_token := csrf_meta.get("content"), str):
-            msg = "Could not find CSRF token on the shindan page."
-            raise ShindanError(msg)
         return get_result_by_ai(
             session,
             params,
             user_inputs=get_user_inputs(source, shindan_name),
             hashtag=hashtag,
-            csrf_token=csrf_token,
+            csrf_token=params["_token"],
             shindan_url=shindan_url,
         )
     if params["type"] == "branch":

{shindan_cli-2.2.1 → shindan_cli-2.2.2}/pyproject.toml

@@ -47,9 +47,9 @@ docs = [
 ]
 
 [tool.hatch]
-build.targets.sdist.include = [ "shindan_cli" ]
-build.targets.wheel.include = [ "shindan_cli" ]
 version.source = "uv-dynamic-versioning"
+build.targets.wheel.include = [ "shindan_cli" ]
+build.targets.sdist.include = [ "shindan_cli" ]
 
 [tool.uv]
 python-downloads = "never"