datasinking 0.2.4__tar.gz → 0.2.5__tar.gz

{datasinking-0.2.4 → datasinking-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasinking
-Version: 0.2.4
+Version: 0.2.5
 Summary: Python client for DataSinking — full-text Asian financial reports (China, Korea, Japan) as Markdown.
 Author: DataSinking
 License: MIT
@@ -111,7 +111,7 @@ coverage, list a company's reports, and extract a figure with correct units.
 
 Every example pulls from the live API and runs as-is.
 
-> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Free keys work too, but fall back to slow per-document fetching (3 req/s, 8,191 documents/day per key, drawn from a 131,071/day pool shared by all free users); a **paid (yearly)** key is strongly recommended for full-exchange downloads.
+> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Quotas count **documents, not requests**, and apply over a rolling 31-day window as well as per day: a free key gets 3 req/s and 8,191 documents/day, inside a pool of 131,071/day and 524,287 per 31 days shared by all free users. A whole exchange will therefore take more than a day on a free key — a **paid (yearly)** key (31 req/s, 131,071 documents/day, 524,287 per 31 days) is strongly recommended.
 
 ## Research (`research/`)
 

{datasinking-0.2.4 → datasinking-0.2.5}/README.md

@@ -89,7 +89,7 @@ coverage, list a company's reports, and extract a figure with correct units.
 
 Every example pulls from the live API and runs as-is.
 
-> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Free keys work too, but fall back to slow per-document fetching (3 req/s, 8,191 documents/day per key, drawn from a 131,071/day pool shared by all free users); a **paid (yearly)** key is strongly recommended for full-exchange downloads.
+> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Quotas count **documents, not requests**, and apply over a rolling 31-day window as well as per day: a free key gets 3 req/s and 8,191 documents/day, inside a pool of 131,071/day and 524,287 per 31 days shared by all free users. A whole exchange will therefore take more than a day on a free key — a **paid (yearly)** key (31 req/s, 131,071 documents/day, 524,287 per 31 days) is strongly recommended.
 
 ## Research (`research/`)
 

datasinking-0.2.5/datasinking/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""DataSinking Python client.
+
+Usage:
+    from datasinking import DataSinking
+    ds = DataSinking("YOUR_API_KEY")
+    ds.list_exchanges()
+
+额度用尽时抛 `QuotaExceeded`（区别于网络/限流错误）:
+    from datasinking import DataSinking, QuotaExceeded
+    try:
+        ds.get_stock_reports("600519.SS", limit=-1)
+    except QuotaExceeded as e:
+        print(e.code, e)   # e.g. quota_month / free_quota_global
+"""
+from ._version import __version__
+from .client import DataSinking, QuotaExceeded
+
+__all__ = ["DataSinking", "QuotaExceeded", "__version__"]

datasinking-0.2.5/datasinking/_version.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""版本号的**唯一来源**。
+
+pyproject.toml 用 `dynamic = ["version"]` 从这里读，`client.py` 的 User-Agent 也读它。
+
+为什么单独抽一个文件：之前 pyproject.toml 和 `__init__.py` 各写各的版本号，
+结果 `__version__` 长期停在 0.1.0 没跟上（发 0.2.4 时才发现，User-Agent 里也一直
+写着旧版本）。现在只有这一处需要改。
+"""
+
+__version__ = "0.2.5"

{datasinking-0.2.4 → datasinking-0.2.5}/datasinking/client.py

@@ -17,11 +17,48 @@ import urllib.request
 import urllib.error
 import urllib.parse
 
+from ._version import __version__
+
+
+class QuotaExceeded(RuntimeError):
+    """额度用尽（HTTP 429 + 服务端返回的 `code`）。
+
+    和「限流」不是一回事：限流等几秒就好（客户端自己会重试），
+    额度类要等**日/月窗口滚动**才有用 —— 重试没有意义，所以直接抛出来，不再重试。
+
+    常见 code：
+      quota_day          年费 key 当日额度用尽（UTC 次日 00:00 恢复）
+      quota_month        年费 key 最近 31 天额度用尽
+      free_quota_key     免费 key 当日额度用尽
+      free_quota_global  免费共享池 当日额度用尽（所有免费用户合计）
+      free_quota_month   免费共享池 最近 31 天额度用尽（所有免费用户合计）
+    """
+
+    def __init__(self, code=None, detail=None):
+        super().__init__(detail or code or "Quota exceeded")
+        self.code = code
+        self.detail = detail
+
+
+def _error_info(e):
+    """从错误响应体里取 (code, detail)。
+
+    服务端把 `code` 留给**非瞬时**的错误：限流（等几秒就好）不带 code，
+    额度类（要等窗口滚动）才带。所以「有没有 code」就是「该不该重试」的判据。
+    """
+    try:
+        body = json.loads(e.read().decode("utf-8"))
+        return body.get("code"), body.get("detail")
+    except Exception:
+        return None, None
+
 
 class DataSinking:
     def __init__(self, api_key, base_url="https://api.datasink.ing"):
         self.base_url = base_url.rstrip("/")
         self.api_key = api_key
+        # batch 探测出来的可用块大小（免费档 3 / 年费 31）。首次调用后缓存，之后不再试错。
+        self._batch_size = None
 
     # ---- 内部: HTTP ----
     def _request(self, method, path, params=None, body=None, retries=5):
@@ -29,7 +66,7 @@ class DataSinking:
         p["apikey"] = self.api_key
         url = f"{self.base_url}{path}?{urllib.parse.urlencode(p)}"
         data = None
-        headers = {"User-Agent": "Mozilla/5.0 (compatible; DataSinking/0.1.0)"}
+        headers = {"User-Agent": f"Mozilla/5.0 (compatible; DataSinking/{__version__})"}
         if body is not None:
             data = json.dumps(body).encode("utf-8")
             headers["Content-Type"] = "application/json"
@@ -40,14 +77,22 @@ class DataSinking:
                 with urllib.request.urlopen(req, timeout=60) as resp:
                     return json.loads(resp.read().decode("utf-8"))
             except urllib.error.HTTPError as e:
-                if e.code == 429:  # 限流, 稍等重试
-                    time.sleep(2)
-                    continue
-                raise  # 其他 HTTP 错误(401/403/404...)直接抛给上层
+                if e.code != 429:
+                    raise  # 401/403/404/400… 直接抛给上层
+                code, detail = _error_info(e)
+                if code:
+                    # 额度类 429：要等日/月窗口滚动，重试没有意义 —— 立刻抛，并带上服务端的原因。
+                    # （以前这里不分青红皂白 sleep(2) 重试，月度额度打满时会空转，
+                    #   最后还抛出一句 "Request failed after retries: None" —— last 变量
+                    #   只在网络异常分支被赋值，429 分支根本不赋值。）
+                    raise QuotaExceeded(code, detail)
+                # 纯限流：等一下再试
+                last = RuntimeError(detail or "Rate limit exceeded (HTTP 429)")
+                time.sleep(2)
             except (urllib.error.URLError, TimeoutError, OSError) as e:
                 time.sleep(1 + i)
                 last = e
-        raise RuntimeError(f"Request failed after retries: {last}")
+        raise RuntimeError(f"Request failed after {retries} retries: {last}")
 
     def _get(self, path, params=None):
         return self._request("GET", path, params)
@@ -67,24 +112,41 @@ class DataSinking:
             page += 1
         return items
 
-    def _batch_content(self, ids, batch_size=31):
-        """批量拉全文; free 计划 batch 返回 403 时自动回退逐篇拉取"""
+    def _batch_content(self, ids, batch_size=None):
+        """批量拉全文。
+
+        服务端**按档位**限制 batch 单次篇数（免费 3 / 年费 31），而客户端事先不知道自己
+        是哪档，所以先按 31 发；被拒（400 / 403）就把块缩小重试，最终退化成逐篇拉取。
+        31 → 10 → 3，免费档试错两次后稳定在 3；**探测出的可用大小会缓存到实例上**，
+        同一个 client 之后的调用不再重复试错。
+
+        旧版这里只认 403，而免费档超限返回的是 **400** —— 会直接 raise 把整个任务打断。
+        """
+        if batch_size is None:
+            batch_size = self._batch_size or 31
         items = []
-        batch_ok = True
-        for i in range(0, len(ids), batch_size):
-            chunk = ids[i : i + batch_size]
-            if batch_ok:
-                try:
-                    b = self._post("/documents/batch", {"doc_ids": chunk})
-                    items.extend(b["items"])
-                    continue
-                except urllib.error.HTTPError as e:
-                    if e.code == 403:
-                        batch_ok = False
-                    else:
-                        raise
-            for did in chunk:
-                items.append(self._get(f"/documents/{did}"))
+        i = 0
+        while i < len(ids):
+            n = min(batch_size, len(ids) - i)
+            chunk = ids[i : i + n]
+
+            if n == 1:  # 块已经缩到 1 篇，batch 没意义，直接走单篇接口
+                items.append(self._get(f"/documents/{chunk[0]}"))
+                i += 1
+                continue
+
+            try:
+                items.extend(self._post("/documents/batch", {"doc_ids": chunk})["items"])
+                self._batch_size = n  # 记住这个大小，下次直接用
+                i += n
+            except QuotaExceeded:
+                raise  # 额度用尽，缩块也没用
+            except urllib.error.HTTPError as e:
+                if e.code not in (400, 403):
+                    raise
+                # 被拒 → 缩块重试。大于 6 就除以 3（收敛快），小了就减一（保证能到 1、
+                # 且不会跳过服务端真实允许的大小，比如 12 → 4 → 3 而不是 12 → 4 → 1）。
+                batch_size = n // 3 if n > 6 else n - 1
         return items
 
     # ---- 5 个核心函数 ----

{datasinking-0.2.4 → datasinking-0.2.5}/datasinking.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasinking
-Version: 0.2.4
+Version: 0.2.5
 Summary: Python client for DataSinking — full-text Asian financial reports (China, Korea, Japan) as Markdown.
 Author: DataSinking
 License: MIT
@@ -111,7 +111,7 @@ coverage, list a company's reports, and extract a figure with correct units.
 
 Every example pulls from the live API and runs as-is.
 
-> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Free keys work too, but fall back to slow per-document fetching (3 req/s, 8,191 documents/day per key, drawn from a 131,071/day pool shared by all free users); a **paid (yearly)** key is strongly recommended for full-exchange downloads.
+> `03_download_exchange.py` fetches every report on an exchange (e.g. all of Shenzhen — 150k+ documents). Quotas count **documents, not requests**, and apply over a rolling 31-day window as well as per day: a free key gets 3 req/s and 8,191 documents/day, inside a pool of 131,071/day and 524,287 per 31 days shared by all free users. A whole exchange will therefore take more than a day on a free key — a **paid (yearly)** key (31 req/s, 131,071 documents/day, 524,287 per 31 days) is strongly recommended.
 
 ## Research (`research/`)
 

{datasinking-0.2.4 → datasinking-0.2.5}/datasinking.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 README.md
 pyproject.toml
 datasinking/__init__.py
+datasinking/_version.py
 datasinking/client.py
 datasinking/mcp_server.py
 datasinking.egg-info/PKG-INFO

{datasinking-0.2.4 → datasinking-0.2.5}/pyproject.toml

@@ -4,7 +4,9 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datasinking"
-version = "0.2.4"
+# 版本号唯一来源 = datasinking/_version.py（client.py 的 User-Agent 也读它）。
+# 以前这里和包内 __version__ 各写各的，结果漂了半年（发 0.2.4 时才发现 __init__ 还停在 0.1.0）。
+dynamic = ["version"]
 description = "Python client for DataSinking — full-text Asian financial reports (China, Korea, Japan) as Markdown."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -32,3 +34,6 @@ Repository = "https://github.com/heubme2020/datasinking"
 
 [tool.setuptools]
 packages = ["datasinking"]
+
+[tool.setuptools.dynamic]
+version = { attr = "datasinking._version.__version__" }

datasinking-0.2.4/datasinking/__init__.py

@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-"""DataSinking Python client.
-
-Usage:
-    from datasinking import DataSinking
-    ds = DataSinking("YOUR_API_KEY")
-    ds.list_exchanges()
-"""
-from .client import DataSinking
-
-# 与 pyproject.toml 的 version 保持一致 —— 这里曾长期停在 0.1.0 没跟着走
-__version__ = "0.2.4"
-__all__ = ["DataSinking"]