xrtm-data 0.7.2__tar.gz → 0.7.4__tar.gz

{xrtm_data-0.7.2/src/xrtm_data.egg-info → xrtm_data-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xrtm-data
-Version: 0.7.2
+Version: 0.7.4
 Summary: The Snapshot Vault for XRTM.
 Author-email: XRTM Team <moy@xrtm.org>
 License-Expression: Apache-2.0

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xrtm-data"
-version = "0.7.2"
+version = "0.7.4"
 description = "The Snapshot Vault for XRTM."
 readme = "README.md"
 requires-python = ">=3.11,<3.13"

xrtm_data-0.7.4/src/xrtm/data/providers/online/_http.py

@@ -0,0 +1,87 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Shared JSON fetching with retry/backoff for online providers.
+
+A single transient network blip used to fail an entire collection pass; these
+providers now retry idempotent GETs a bounded number of times.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional
+
+from xrtm.data.core import DataSourceError
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["fetch_json"]
+
+DEFAULT_USER_AGENT = "xrtm/0.1"
+RETRYABLE_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504})
+
+
+def fetch_json(
+    url: str,
+    *,
+    timeout: float = 15.0,
+    retries: int = 2,
+    headers: Optional[Dict[str, str]] = None,
+    provider: str = "API",
+) -> Any:
+    r"""GET *url* and parse JSON, retrying transient failures with backoff.
+
+    Client errors (4xx other than 408/409/425/429) are not retried — they will
+    fail again and retrying them just burns time.
+    """
+    request = urllib.request.Request(url, headers={"User-Agent": DEFAULT_USER_AGENT, **(headers or {})})
+    last_error: Optional[Exception] = None
+
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            last_error = exc
+            if exc.code not in RETRYABLE_STATUS or attempt >= retries:
+                raise DataSourceError(f"{provider} API error: {exc}") from exc
+        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+            # The body was read in full but does not parse: deterministic, so
+            # retrying only multiplies the damage (e.g. an HTML maintenance
+            # page served with HTTP 200).
+            raise DataSourceError(f"{provider} API returned unparseable JSON: {exc}") from exc
+        except Exception as exc:  # noqa: BLE001 - connection/timeout errors are transient
+            last_error = exc
+            if attempt >= retries:
+                raise DataSourceError(f"{provider} API error: {exc}") from exc
+
+        wait = (0.5 * (2**attempt)) + random.uniform(0.0, 0.25)
+        logger.warning(
+            "%s request failed (attempt %d/%d), retrying in %.1fs: %s",
+            provider,
+            attempt + 1,
+            retries,
+            wait,
+            last_error,
+        )
+        time.sleep(wait)
+
+    raise DataSourceError(f"{provider} API error: {last_error}")  # pragma: no cover - loop raises

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/src/xrtm/data/providers/online/kalshi.py

@@ -23,15 +23,14 @@ API docs: https://trading-api.readme.io/
 
 from __future__ import annotations
 
-import json
 import logging
-import urllib.request
 from datetime import datetime, timezone
 from typing import Any, List, Optional
 
-from xrtm.data.core.interfaces import DataSource, DataSourceError, ResolutionSource
+from xrtm.data.core.interfaces import DataSource, ResolutionSource
 from xrtm.data.core.schemas.forecast import ForecastQuestion, MetadataBase
 from xrtm.data.core.schemas.resolution import ResolvedQuestion
+from xrtm.data.providers.online._http import fetch_json
 
 logger = logging.getLogger(__name__)
 
@@ -148,13 +147,8 @@ class KalshiSource(DataSource, ResolutionSource):
         return resolved
 
     def _get_json(self, url: str) -> Any:
-        r"""Fetch JSON from a URL."""
-        req = urllib.request.Request(url, headers={"User-Agent": "xrtm/0.1"})
-        try:
-            with urllib.request.urlopen(req, timeout=20) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except Exception as exc:
-            raise DataSourceError(f"Kalshi API error: {exc}") from exc
+        r"""Fetch JSON from a URL (retry/backoff on transient failures)."""
+        return fetch_json(url, timeout=20, provider="Kalshi")
 
     @staticmethod
     def _to_forecast_question(item: dict[str, Any]) -> ForecastQuestion:

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/src/xrtm/data/providers/online/manifold.py

@@ -23,15 +23,14 @@ API docs: https://docs.manifold.markets/api
 
 from __future__ import annotations
 
-import json
 import logging
-import urllib.request
 from datetime import datetime, timezone
 from typing import Any, List, Optional
 
-from xrtm.data.core.interfaces import DataSource, DataSourceError, ResolutionSource
+from xrtm.data.core.interfaces import DataSource, ResolutionSource
 from xrtm.data.core.schemas.forecast import ForecastQuestion, MetadataBase
 from xrtm.data.core.schemas.resolution import ResolvedQuestion
+from xrtm.data.providers.online._http import fetch_json
 
 logger = logging.getLogger(__name__)
 
@@ -149,13 +148,8 @@ class ManifoldSource(DataSource, ResolutionSource):
         return resolved
 
     def _get_json(self, url: str) -> Any:
-        r"""Fetch JSON from a URL."""
-        req = urllib.request.Request(url, headers={"User-Agent": "xrtm/0.1"})
-        try:
-            with urllib.request.urlopen(req, timeout=20) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except Exception as exc:
-            raise DataSourceError(f"Manifold API error: {exc}") from exc
+        r"""Fetch JSON from a URL (retry/backoff on transient failures)."""
+        return fetch_json(url, timeout=20, provider="Manifold")
 
     @staticmethod
     def _to_forecast_question(item: dict[str, Any]) -> ForecastQuestion:

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/src/xrtm/data/providers/online/metaculus.py

@@ -25,15 +25,14 @@ API docs: https://www.metaculus.com/api2/
 
 from __future__ import annotations
 
-import json
 import logging
-import urllib.request
 from datetime import datetime, timezone
 from typing import Any
 
 from xrtm.data.core.interfaces import DataSource, DataSourceError, ResolutionSource
 from xrtm.data.core.schemas.forecast import ForecastQuestion, MetadataBase
 from xrtm.data.core.schemas.resolution import ResolvedQuestion
+from xrtm.data.providers.online._http import fetch_json
 
 logger = logging.getLogger(__name__)
 
@@ -198,16 +197,11 @@ class MetaculusSource(DataSource, ResolutionSource):
             return datetime.now(timezone.utc)
 
     def _get_json(self, url: str) -> dict[str, Any]:
-        r"""Fetch JSON from a URL."""
-        headers = {"User-Agent": "xrtm/0.1"}
+        r"""Fetch JSON from a URL (retry/backoff on transient failures)."""
+        headers: dict[str, str] = {}
         if self.api_key:
             headers["Authorization"] = f"Token {self.api_key}"
-        req = urllib.request.Request(url, headers=headers)
-        try:
-            with urllib.request.urlopen(req, timeout=15) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except Exception as exc:
-            raise DataSourceError(f"Metaculus API error: {exc}") from exc
+        return fetch_json(url, timeout=15, headers=headers, provider="Metaculus")
 
     @staticmethod
     def _to_forecast_question(item: dict[str, Any]) -> ForecastQuestion:

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/src/xrtm/data/providers/online/polymarket.py

@@ -25,13 +25,13 @@ from __future__ import annotations
 
 import json
 import logging
-import urllib.request
 from datetime import datetime, timezone
 from typing import Any
 
 from xrtm.data.core.interfaces import DataSource, DataSourceError, ResolutionSource
 from xrtm.data.core.schemas.forecast import ForecastQuestion, MetadataBase
 from xrtm.data.core.schemas.resolution import ResolvedQuestion
+from xrtm.data.providers.online._http import fetch_json
 
 logger = logging.getLogger(__name__)
 
@@ -80,8 +80,8 @@ class PolymarketSource(DataSource, ResolutionSource):
             data = self._get_json(url)
         except DataSourceError:
             # Fallback: same markets endpoint without the volume ordering,
-            # which some Gamma deployments reject.
-            url = f"{self.api_base}/markets?limit={min(limit, 100)}&closed=false"
+            # which some Gamma deployments reject. Pagination still applies.
+            url = f"{self.api_base}/markets?limit={min(limit, 100)}&offset={offset}&closed=false"
             data = self._get_json(url)
 
         # Gamma API may wrap in "markets" key or return list directly
@@ -199,13 +199,8 @@ class PolymarketSource(DataSource, ResolutionSource):
             return datetime.now(timezone.utc)
 
     def _get_json(self, url: str) -> Any:
-        r"""Fetch JSON from a URL."""
-        req = urllib.request.Request(url, headers={"User-Agent": "xrtm/0.1"})
-        try:
-            with urllib.request.urlopen(req, timeout=15) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except Exception as exc:
-            raise DataSourceError(f"Polymarket API error: {exc}") from exc
+        r"""Fetch JSON from a URL (retry/backoff on transient failures)."""
+        return fetch_json(url, timeout=15, provider="Polymarket")
 
     @staticmethod
     def _to_forecast_question(item: dict[str, Any]) -> ForecastQuestion:

{xrtm_data-0.7.2 → xrtm_data-0.7.4/src/xrtm_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xrtm-data
-Version: 0.7.2
+Version: 0.7.4
 Summary: The Snapshot Vault for XRTM.
 Author-email: XRTM Team <moy@xrtm.org>
 License-Expression: Apache-2.0

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/src/xrtm_data.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ src/xrtm/data/corpora/real_binary.py
 src/xrtm/data/corpora/registry.py
 src/xrtm/data/providers/__init__.py
 src/xrtm/data/providers/online/__init__.py
+src/xrtm/data/providers/online/_http.py
 src/xrtm/data/providers/online/kalshi.py
 src/xrtm/data/providers/online/manifold.py
 src/xrtm/data/providers/online/metaculus.py

{xrtm_data-0.7.2 → xrtm_data-0.7.4}/tests/test_online_sources.py

@@ -229,3 +229,101 @@ async def test_polymarket_fetch_questions_fallback_uses_markets(monkeypatch):
     assert len(calls["urls"]) == 2
     assert "/markets?" in calls["urls"][1]
     assert questions and questions[0].id == "polymarket-m1"
+
+
+def test_fetch_json_retries_transient_failures(monkeypatch):
+    from xrtm.data.providers.online import _http
+
+    calls = {"n": 0}
+
+    class _Response:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+        def read(self):
+            return b'{"ok": true}'
+
+    def _urlopen(request, timeout=None):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise OSError("connection reset")
+        return _Response()
+
+    monkeypatch.setattr(_http.urllib.request, "urlopen", _urlopen)
+    monkeypatch.setattr(_http.time, "sleep", lambda _seconds: None)
+
+    assert _http.fetch_json("https://example.test/x") == {"ok": True}
+    assert calls["n"] == 3
+
+
+def test_fetch_json_does_not_retry_client_errors(monkeypatch):
+    import urllib.error
+
+    from xrtm.data import DataSourceError
+    from xrtm.data.providers.online import _http
+
+    calls = {"n": 0}
+
+    def _urlopen(request, timeout=None):
+        calls["n"] += 1
+        raise urllib.error.HTTPError(request.full_url, 404, "Not Found", {}, None)
+
+    monkeypatch.setattr(_http.urllib.request, "urlopen", _urlopen)
+    monkeypatch.setattr(_http.time, "sleep", lambda _seconds: None)
+
+    with pytest.raises(DataSourceError):
+        _http.fetch_json("https://example.test/x")
+    assert calls["n"] == 1
+
+
+def test_fetch_json_does_not_retry_unparseable_bodies(monkeypatch):
+    from xrtm.data.providers.online import _http
+
+    calls = {"n": 0}
+
+    class _Response:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+        def read(self):
+            return b"<html>maintenance</html>"
+
+    def _urlopen(request, timeout=None):
+        calls["n"] += 1
+        return _Response()
+
+    monkeypatch.setattr(_http.urllib.request, "urlopen", _urlopen)
+    monkeypatch.setattr(_http.time, "sleep", lambda _seconds: None)
+
+    from xrtm.data import DataSourceError
+
+    with pytest.raises(DataSourceError, match="unparseable"):
+        _http.fetch_json("https://example.test/x")
+    assert calls["n"] == 1  # deterministic parse failure: no retries
+
+
+@pytest.mark.asyncio
+async def test_polymarket_fallback_keeps_offset(monkeypatch):
+    from xrtm.data.core import DataSourceError
+
+    source = PolymarketSource()
+    calls = {"urls": []}
+
+    def _get_json(url):
+        calls["urls"].append(url)
+        if "order=volume24hr" in url:
+            raise DataSourceError("volume ordering unsupported")
+        return []
+
+    monkeypatch.setattr(source, "_get_json", _get_json)
+
+    await source.fetch_questions(limit=5, offset=20)
+
+    assert len(calls["urls"]) == 2
+    assert "offset=20" in calls["urls"][1]