shindan-cli 2.2.0__tar.gz → 2.2.2__tar.gz

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: shindan-cli
-Version: 2.2.0
+Version: 2.2.2
 Summary: ShindanMaker (https://shindanmaker.com) CLI
 Project-URL: Repository, https://github.com/eggplants/shindan-cli
 Author-email: eggplants <w10776e8w@yahoo.co.jp>
-License: MIT
+License-Expression: MIT
 License-File: LICENSE
 Keywords: cli,shindanmaker
 Classifier: Development Status :: 3 - Alpha
@@ -57,7 +57,7 @@ Supported types of diagnosis:
 
 - [Name-based diagnosis (名前診断)](https://shindanmaker.com/list/name)
 - [Branching diagnosis (分岐診断)](https://shindanmaker.com/list/branch)
-- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai)
+- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai) (currently blocked by a Cloudflare Turnstile challenge)
 - [Check Diagnosis (チェック診断)](https://shindanmaker.com/list/check)
 
 ### CLI

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/README.md

@@ -32,7 +32,7 @@ Supported types of diagnosis:
 
 - [Name-based diagnosis (名前診断)](https://shindanmaker.com/list/name)
 - [Branching diagnosis (分岐診断)](https://shindanmaker.com/list/branch)
-- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai)
+- [AI diagnosis (AI診断)](https://shindanmaker.com/list/ai) (currently blocked by a Cloudflare Turnstile challenge)
 - [Check Diagnosis (チェック診断)](https://shindanmaker.com/list/check)
 
 ### CLI

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/pyproject.toml

@@ -1,6 +1,5 @@
 [build-system]
 build-backend = "hatchling.build"
-
 requires = [ "hatchling", "uv-dynamic-versioning" ]
 
 [project]
@@ -11,7 +10,7 @@ keywords = [
   "cli",
   "shindanmaker",
 ]
-license = { text = "MIT" }
+license = "MIT"
 authors = [ { name = "eggplants", email = "w10776e8w@yahoo.co.jp" } ]
 requires-python = ">=3.10,<4"
 classifiers = [
@@ -32,7 +31,6 @@ dependencies = [
   "lxml>=5.3,<7",
   "requests>=2.32.3,<3",
 ]
-
 urls.Repository = "https://github.com/eggplants/shindan-cli"
 scripts.shindan = "shindan_cli.main:main"
 
@@ -48,22 +46,19 @@ docs = [
   "pdoc>=16",
 ]
 
-[tool.hatch.version]
-source = "uv-dynamic-versioning"
-
-[tool.hatch.build.targets.sdist]
-include = [ "shindan_cli" ]
-
-[tool.hatch.build.targets.wheel]
-include = [ "shindan_cli" ]
+[tool.hatch]
+version.source = "uv-dynamic-versioning"
+build.targets.wheel.include = [ "shindan_cli" ]
+build.targets.sdist.include = [ "shindan_cli" ]
 
 [tool.uv]
-preview = true
+python-downloads = "never"
 default-groups = [
   "dev",
   "docs",
 ]
 exclude-newer = "P7D"
+preview = true
 
 [tool.ruff]
 line-length = 120

shindan_cli-2.2.2/shindan_cli/_http.py

@@ -0,0 +1,79 @@
+"""HTTP helpers to cope with rate limiting on <https://shindanmaker.com>."""
+
+from __future__ import annotations
+
+import random
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from requests import Response, Session
+
+TOO_MANY_REQUESTS = 429
+
+MAX_RETRIES = 5
+INITIAL_WAIT = 2.0
+BACKOFF_FACTOR = 2.0
+
+WAIT_RANGE = (2.0, 5.0)
+
+
+def random_wait() -> None:
+    """Sleep for a random while, to fetch at a less machine-like pace."""
+    time.sleep(random.uniform(*WAIT_RANGE))  # noqa: S311
+
+
+def backoff_wait(attempt: int) -> float:
+    """Get how long to wait before the given retry attempt.
+
+    Args:
+        attempt (int): 0-based index of the retry about to be made
+
+    Returns:
+        float: seconds to sleep
+
+    """
+    return INITIAL_WAIT * BACKOFF_FACTOR**attempt
+
+
+def request_with_retry(
+    session: Session,
+    method: str,
+    url: str,
+    **kwargs: Any,  # noqa: ANN401
+) -> Response:
+    """Send a request, retrying with a backoff while the site rate limits us.
+
+    The site is fronted by Cloudflare, which answers bursts of requests with a
+    `429` challenge page instead of the result. Those responses are transient,
+    so they are retried rather than reported to the caller.
+
+    Args:
+        session (Session): session object
+        method (str): HTTP method
+        url (str): url to request
+        **kwargs (Any): extra arguments passed to `Session.request`
+
+    Returns:
+        Response: the last response received
+
+    """
+    response = session.request(method, url, **kwargs)
+    for attempt in range(MAX_RETRIES - 1):
+        if response.status_code != TOO_MANY_REQUESTS:
+            break
+        time.sleep(backoff_wait(attempt))
+        response = session.request(method, url, **kwargs)
+    return response
+
+
+__all__ = (
+    "BACKOFF_FACTOR",
+    "INITIAL_WAIT",
+    "MAX_RETRIES",
+    "TOO_MANY_REQUESTS",
+    "WAIT_RANGE",
+    "backoff_wait",
+    "random_wait",
+    "request_with_retry",
+)

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/shindan_cli/constants.py

@@ -7,34 +7,26 @@ class AIParams(TypedDict):
     _token: str
     randname: str
     type: Literal["ai"]
-    shindan_token: str
     encrypted_exec_key: str
 
 
 class BranchParams(TypedDict):
     _token: str
     randname: str
-    hiddenName: str
     type: Literal["branch"]
-    shindan_token: str
     rbr: str
 
 
 class CheckParams(TypedDict):
     _token: str
     randname: str
-    hiddenName: str
     type: Literal["check"]
-    shindan_token: str
-    # input-check-choice[choice_id]: str
 
 
 class NameParams(TypedDict):
     _token: str
     randname: str
-    hiddenName: str
     type: Literal["name"]
-    shindan_token: str
 
 
 class TargetKeysByType(TypedDict):

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/shindan_cli/get_results.py

@@ -3,18 +3,20 @@
 from __future__ import annotations
 
 import re
-from typing import TYPE_CHECKING, Union
+import time
+from typing import TYPE_CHECKING, cast
 
 from bs4 import BeautifulSoup, Tag
 
-from .constants import BASE_URL, HEADERS, AIParams, BranchParams, CheckParams, NameParams
+from ._http import MAX_RETRIES, backoff_wait, request_with_retry
+from .constants import HEADERS, AIParams, BranchParams, CheckParams, NameParams
 
 if TYPE_CHECKING:
     from requests import Session
 
     from .models import ShindanResult, UserInputs
 
-Params = Union[AIParams, BranchParams, CheckParams, NameParams]
+Params = AIParams | BranchParams | CheckParams | NameParams
 
 
 def __get_result(
@@ -24,16 +26,28 @@ def __get_result(
     is_renewal: bool = False,
     shindan_url: str,
 ) -> ShindanResult:
-    result_page = session.post(
-        shindan_url + ("/r" if is_renewal else ""),
-        data=params,
-        headers=HEADERS,
-    )
-    soup = BeautifulSoup(result_page.text, features="lxml")
-    result_tag = soup.find(id="share-copytext-shindanresult-textarea")
-
-    if not isinstance(result_tag, Tag) or not result_tag.text:
-        msg = f"Could not find a tag contains the result, returns: {result_tag}"
+    result_tag: Tag | None = None
+    status_code = None
+
+    for attempt in range(MAX_RETRIES):
+        if attempt > 0:
+            time.sleep(backoff_wait(attempt - 1))
+        result_page = request_with_retry(
+            session,
+            "POST",
+            shindan_url + ("/r" if is_renewal else ""),
+            data=params,
+            headers=HEADERS,
+        )
+        status_code = result_page.status_code
+        soup = BeautifulSoup(result_page.text, features="lxml")
+        found = soup.find(id="share-copytext-shindanresult-textarea")
+        if isinstance(found, Tag) and found.text:
+            result_tag = found
+            break
+
+    if result_tag is None:
+        msg = f"Could not find a tag contains the result, last status code: {status_code}"
         raise TypeError(msg)
 
     *results, hashtag, shindan_url, _ = result_tag.text.split("\n")
@@ -54,6 +68,7 @@ def get_result_by_ai(
     *,
     user_inputs: UserInputs,
     hashtag: str | None,
+    csrf_token: str,
     shindan_url: str,
 ) -> ShindanResult:
     """Get result by AI type shindan.
@@ -63,36 +78,18 @@ def get_result_by_ai(
         params (Params): input parameters fetched from shindan page
         user_inputs (UserInputs): user inputs
         hashtag (str | None): hashtag
+        csrf_token (str): CSRF token extracted from the shindan page
         shindan_url (str): shindan url
 
     Returns:
         ShindanResult: the returned result from <https://shindanmaker.com>
 
     """
-    res = session.post(
-        shindan_url,
-        data=params,
-        headers=HEADERS,
-    )
-    meta = BeautifulSoup(
-        res.text,
-        features="lxml",
-    ).select_one('meta[name="csrf-token"]')
-    if not res.ok or not meta or not isinstance(csrf_token := meta.get("content"), str):
-        msg = f"Failed to get the csrf token. ({res.status_code})"
-        raise ValueError(msg)
-    HEADERS.update({"x-csrf-token": csrf_token})
-    if not res.ok:
-        res = session.post(
-            f"{BASE_URL}/ai_life_update",
-            data={"ai_life": 3},
-            headers=HEADERS,
-        )
-        if not res.ok:
-            msg = f"Failed to update AI life. ({res.status_code})"
-            raise ValueError(msg)
+    ai_headers = {**HEADERS, "x-csrf-token": csrf_token}
 
-    result_sse = session.post(
+    result_sse = request_with_retry(
+        session,
+        "POST",
         f"{shindan_url}/ai_result",
         json={
             "form_values": user_inputs,
@@ -100,7 +97,7 @@ def get_result_by_ai(
             "ai_result_request_times": 0,
             "encrypted_exec_key": params["encrypted_exec_key"],
         },
-        headers=HEADERS,
+        headers=ai_headers,
     )
     gpt_results = "".join(
         re.findall(r'"content":"([^"]+)', result_sse.text),
@@ -151,8 +148,9 @@ def get_result_by_check(
         ShindanResult: the returned result from <https://shindanmaker.com>
 
     """
+    check_params = cast("dict[str, str]", params)
     for choice_id, answer_id in user_choices.items():
-        params[f"input-check-choice[{choice_id}]"] = answer_id  # type: ignore[literal-required]
+        check_params[f"input-check-choice[{choice_id}]"] = answer_id
 
     return __get_result(session, params, is_renewal=True, shindan_url=shindan_url)
 

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/shindan_cli/interactive.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from shindan_cli.models import QuestionBranchChoice, QuestionChoiceChoice, UserInput, QuestionBranch, QuestionChoice
+from shindan_cli.models import QuestionBranch, QuestionBranchChoice, QuestionChoice, QuestionChoiceChoice, UserInput
 
 if TYPE_CHECKING:
     from bs4 import BeautifulSoup
@@ -27,7 +27,7 @@ def get_user_inputs(
 
     """
     user_inputs: dict[str, UserInput] = {}
-    form_labels = source.select("form#shindanForm > div.px-3 > div > span")
+    form_labels = source.select("form#shindanForm label")
     for idx, question in enumerate([*form_labels, *range(10 - len(form_labels))]):
         if isinstance(question, int):
             user_inputs[f"user_input_{idx + 1}"] = UserInput({"q": "", "a": None})

{shindan_cli-2.2.0 → shindan_cli-2.2.2}/shindan_cli/shindan.py

@@ -2,14 +2,14 @@
 
 from __future__ import annotations
 
-import random
-import time
 from typing import cast
 
 import cloudscraper  # type: ignore[unused-ignore,import-not-found,import-untyped]
 from bs4 import BeautifulSoup
 from requests.sessions import Session
 
+from . import _http
+from ._http import request_with_retry
 from .constants import BASE_URL, HEADERS, TARGET_KEYS_BY_TYPE
 from .get_results import (
     Params,
@@ -26,6 +26,42 @@ class ShindanError(Exception):
     """Error class for shindan-cli."""
 
 
+def __get_csrf_token(session: Session, source: BeautifulSoup) -> str:
+    """Get a CSRF token to submit the shindan form with.
+
+    The shindan pages are served from a cache with their token fields left
+    blank, and the browser fills them in from `/csrf-token` just before
+    submitting the form.
+
+    Args:
+        session (Session): session object
+        source (BeautifulSoup): parsed shindan page
+
+    Returns:
+        str: CSRF token
+
+    Raises:
+        ShindanError
+
+    """
+    csrf_meta = source.select_one('meta[name="csrf-token"]')
+    if csrf_meta and isinstance(token := csrf_meta.get("content"), str) and token:
+        return token
+
+    response = request_with_retry(
+        session,
+        "GET",
+        f"{BASE_URL}/csrf-token",
+        headers={**HEADERS, "X-Requested-With": "XMLHttpRequest"},
+    )
+    if response.status_code != 200:  # noqa: PLR2004
+        raise ShindanError(response.status_code)
+    if not isinstance(token := response.json().get("token"), str) or not token:
+        msg = "Could not find CSRF token on the shindan page."
+        raise ShindanError(msg)
+    return token
+
+
 def shindan(
     page_id: int,
     shindan_name: str,
@@ -55,10 +91,10 @@ def shindan(
     shindan_url = f"{BASE_URL}/{page_id}"
 
     session = cloudscraper.create_scraper()
-    assert isinstance(session, Session)
+    assert isinstance(session, Session)  # noqa: S101
 
-    shindan_page = session.get(shindan_url, headers=HEADERS)
-    if shindan_page.status_code != 200:
+    shindan_page = request_with_retry(session, "GET", shindan_url, headers=HEADERS)
+    if shindan_page.status_code != 200:  # noqa: PLR2004
         raise ShindanError(shindan_page.status_code)
 
     source = BeautifulSoup(shindan_page.text, features="lxml")
@@ -78,9 +114,10 @@ def shindan(
     )
     # overwrite randname (old: shindanName)
     params["randname"] = shindan_name
+    params["_token"] = params["_token"] or __get_csrf_token(session, source)
 
     if wait:
-        time.sleep(random.uniform(2, 5))  # noqa: S311
+        _http.random_wait()
 
     if params["type"] == "ai":
         hashtag_title = source.select_one("h1#shindanTitle")
@@ -94,6 +131,7 @@ def shindan(
             params,
             user_inputs=get_user_inputs(source, shindan_name),
             hashtag=hashtag,
+            csrf_token=params["_token"],
             shindan_url=shindan_url,
         )
     if params["type"] == "branch":