dataquery-sdk 1.2.0__tar.gz → 1.2.2__tar.gz

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/CHANGELOG.md

@@ -40,5 +40,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - SSE: jittered reconnect backoff to avoid synchronized reconnect storms; stop reconnecting on fatal 403/404 and bound retries on 401; idle `sock_read` timeout distinguished from the heartbeat watchdog; honor the server `retry:` hint; strip a leading UTF-8 BOM; larger read buffer guards against `LineTooLong`; `stop()` is await-safe under concurrent callers
 ## [1.2.0] - 2026-06-29
 - DataQuery functions: new `function-help` command for local lookup of all 158 DQ function syntaxes, parameters, and categories (no API call); backed by a static, frozen `dataquery/data/function.json` dataset
+## [1.2.1] - 2026-07-14
+- Written research: new `download_zip_async` helper that downloads a group over a date range (split into calendar-month windows to fit the available-files endpoint limit) and safely extracts ZIP archives as each download completes, overlapping unzip with in-flight downloads
+- Group downloads: `run_group_download_async` accepts an `on_file_complete` async callback awaited per file on `completed`/`already_exists`
+- Extraction is Zip Slip-guarded, skips current-day archives, and surfaces failures via `extraction_errors` (downgrading overall status to `partial`); date windows with no available files no longer mark a multi-window run as `partial`
+## [1.2.2] - 2026-07-18
+- MCP: new `mcp-connect` CLI command  
 
 

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dataquery-sdk
-Version: 1.2.0
+Version: 1.2.2
 Summary: Python SDK for DATAQUERY Data API - Query, download, and check availability of economic data files
 Author-email: DATAQUERY SDK Team <dataquery_support@jpmorgan.com>
-Project-URL: Homepage, https://github.com/dataquery/dataquery-sdk
-Project-URL: Bug Tracker, https://github.com/dataquery/dataquery-sdk/issues
-Project-URL: Documentation, https://github.com/dataquery/dataquery-sdk/wiki
-Project-URL: Source Code, https://github.com/dataquery/dataquery-sdk
+Project-URL: Homepage, https://github.com/jpmorganchase/dataquery-sdk
+Project-URL: Bug Tracker, https://github.com/jpmorganchase/dataquery-sdk/issues
+Project-URL: Documentation, https://github.com/jpmorganchase/dataquery-sdk/wiki
+Project-URL: Source Code, https://github.com/jpmorganchase/dataquery-sdk
 Keywords: dataquery,data,api,economic,financial,download,async,oauth,rate-limiting,sdk
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
@@ -49,6 +49,8 @@ Requires-Dist: mike>=1.1.0; extra == "docs"
 Requires-Dist: pymdown-extensions>=10.21.3; extra == "docs"
 Provides-Extra: pandas
 Requires-Dist: pandas>=2.0.0; extra == "pandas"
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=2.14; extra == "mcp"
 Provides-Extra: all
 Requires-Dist: pytest>=7.0.0; extra == "all"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "all"
@@ -61,6 +63,7 @@ Requires-Dist: mkdocs-material>=9.0.0; extra == "all"
 Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "all"
 Requires-Dist: mike>=1.1.0; extra == "all"
 Requires-Dist: pandas>=2.0.0; extra == "all"
+Requires-Dist: fastmcp>=2.14; extra == "all"
 Requires-Dist: pip-audit>=2.7.0; extra == "all"
 Requires-Dist: build>=1.3.0; extra == "all"
 Requires-Dist: urllib3>=2.7.0; extra == "all"

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/__init__.py

@@ -11,10 +11,10 @@ Quick Start:
     ...     groups = await dq.list_groups_async()
     ...     print(f"Found {len(groups)} groups")
 
-For more information, visit: https://github.com/dataquery/dataquery-sdk
+For more information, visit: https://github.com/jpmorganchase/dataquery-sdk
 """
 
-__version__ = "1.2.0"
+__version__ = "1.2.2"
 __author__ = "DATAQUERY SDK Team"
 __email__ = "dataquery_support@jpmorgan.com"
 __license__ = "MIT"
@@ -63,6 +63,7 @@ from .types.models import (
     Group,
     GroupList,
 )
+from .utils import download_zip_async
 
 __all__ = [
     "DataQuery",
@@ -103,6 +104,7 @@ __all__ = [
     "get_function_registry",
     "list_functions_by_category",
     "lookup_function",
+    "download_zip_async",
 ]
 
 __version_info__ = tuple(int(x) for x in __version__.split("."))

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/cli.py

@@ -4,6 +4,7 @@ import argparse
 import asyncio
 import json
 import sys
+from collections.abc import AsyncGenerator
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -183,7 +184,7 @@ def create_parser() -> argparse.ArgumentParser:
     _ = auth_sub.add_parser("test", help="Test authentication by listing groups")
 
     p_search = subparsers.add_parser(
-        "text-search",
+        "search",
         help="Search the DataQuery catalog using a natural-language query (POST /search)",
     )
     p_search.add_argument("--query", required=True, help="Free-text search query")
@@ -290,6 +291,20 @@ def create_parser() -> argparse.ArgumentParser:
 
     subparsers.add_parser("heartbeat", help="Check if DataQuery is running")
 
+    p_connect = subparsers.add_parser(
+        "mcp-connect",
+        help="Connect a local stdio MCP client to a remote MCP server (AuthE OAuth)",
+        description=(
+            "Bridge a desktop MCP client (stdio) to a remote streamable-HTTP MCP\n"
+            "server, authenticating with an OAuth client-credentials (AuthE) token\n"
+            "minted from the DATAQUERY_* environment. Point your MCP client's\n"
+            "`command` at:  dataquery mcp-connect --url <MCP_URL>"
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_connect.add_argument("--url", required=True, help="Remote MCP endpoint URL")
+    p_connect.add_argument("--name", default="dataquery-mcp", help="Proxy server name (default: dataquery-mcp)")
+
     return parser
 
 
@@ -483,9 +498,9 @@ async def cmd_auth_test(args: argparse.Namespace) -> int:
     return 0
 
 
-async def cmd_text_search(args: argparse.Namespace) -> int:
+async def cmd_search(args: argparse.Namespace) -> int:
     async with DataQuery(args.env_file) as dq:
-        result = await dq.text_search_async(args.query)
+        result = await dq.search_async(args.query)
 
     if args.json:
         print(json.dumps(result, indent=2))
@@ -716,6 +731,56 @@ def cmd_function_help(args: argparse.Namespace) -> int:
     return 0
 
 
+async def cmd_mcp_connect(args: argparse.Namespace) -> int:
+    """Bridge a desktop MCP client (stdio) to a remote streamable-HTTP MCP server.
+
+    Mints an OAuth client-credentials (AuthE) token with the SDK's own
+    TokenManager from the ``DATAQUERY_*`` environment and attaches a fresh bearer
+    token to every upstream request. Launched by an MCP client as its stdio
+    ``command``; stdout carries only the MCP JSON-RPC stream, so all diagnostics
+    go to stderr.
+    """
+    try:
+        import httpx
+        from fastmcp import FastMCP
+        from fastmcp.client.transports import StreamableHttpTransport
+    except ImportError:
+        print(
+            "The MCP bridge requires the 'mcp' extra. Install it with:\n"
+            "    pip install 'dataquery-sdk[mcp]'\n"
+            "or run it directly with:\n"
+            "    uvx --from 'dataquery-sdk[mcp]' dataquery mcp-connect --url <MCP_URL>",
+            file=sys.stderr,
+        )
+        return 1
+
+    from dataquery.config import EnvConfig
+    from dataquery.transport.auth import TokenManager
+
+    config = EnvConfig.create_client_config(env_file=Path(args.env_file) if getattr(args, "env_file", None) else None)
+    token_manager = TokenManager(config)
+
+    class _AutheAuth(httpx.Auth):
+        """Stamp a fresh AuthE bearer token (from the SDK TokenManager) per request."""
+
+        async def async_auth_flow(self, request: httpx.Request) -> AsyncGenerator[httpx.Request, httpx.Response]:
+            header = await token_manager.get_valid_token()  # "Bearer <jwt>"
+            if not header:
+                raise DataQueryError(
+                    "Could not obtain an OAuth token \u2014 check DATAQUERY_CLIENT_ID, "
+                    "DATAQUERY_CLIENT_SECRET, DATAQUERY_OAUTH_TOKEN_URL and "
+                    "DATAQUERY_OAUTH_AUD."
+                )
+            request.headers["Authorization"] = header
+            yield request
+
+    transport = StreamableHttpTransport(args.url, auth=_AutheAuth())
+    proxy = FastMCP.as_proxy(transport, name=args.name)
+    # stdout is reserved for the MCP JSON-RPC stream; keep the banner off it.
+    await proxy.run_async(transport="stdio", show_banner=False)
+    return 0
+
+
 def main_sync(ns: argparse.Namespace) -> int:
     if ns.command == "config":
         if ns.config_command == "show":
@@ -734,7 +799,7 @@ _ASYNC_COMMANDS = {
     "availability": cmd_availability,
     "download": cmd_download,
     "download-group": cmd_download_group,
-    "text-search": cmd_text_search,
+    "search": cmd_search,
     "groups-search": cmd_groups_search,
     "instruments": cmd_instruments,
     "instruments-search": cmd_instruments_search,
@@ -745,6 +810,7 @@ _ASYNC_COMMANDS = {
     "expression-timeseries": cmd_expression_timeseries,
     "grid-data": cmd_grid_data,
     "heartbeat": cmd_heartbeat,
+    "mcp-connect": cmd_mcp_connect,
 }
 
 

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/config/env.py

@@ -247,7 +247,7 @@ class EnvConfig:
         if config.oauth_enabled:
             if not config.client_id:
                 errors.append("CLIENT_ID is required when OAuth is enabled")
-            if not config.client_secret:
+            if not config.get_client_secret():
                 errors.append("CLIENT_SECRET is required when OAuth is enabled")
             if not config.oauth_token_url:
                 errors.append("OAUTH_TOKEN_URL is required when OAuth is enabled")

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/constants/download.py

@@ -28,6 +28,13 @@ MBPS_TO_BYTES_PER_SECOND = 125_000
 
 # Default size, in days, of each chunk when splitting a date range so that
 # the available-files endpoint is queried over a smaller window per call.
-# Used by ``run_group_download_chunked_async`` to honour group-level
-# date-range limits (e.g. RESEARCH_EQUITY_ALL is restricted to 14 days).
+# Used by ``split_date_range_into_chunks``. The endpoint caps each call at
+# one calendar month; smaller windows just keep responses lighter. (The old
+# group-level limits, e.g. 14 days for RESEARCH_EQUITY_ALL, were lifted
+# server-side.)
 DEFAULT_WRITTEN_RESEARCH_CHUNK_DAYS: int = 7
+
+# Error string set on the OperationReport when the available-files endpoint
+# returns nothing for a date window. Chunked workflows match on it to tell
+# a quiet window apart from a real failure.
+NO_FILES_FOUND_ERROR: str = "No available files found for date range"

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/core/_mixins.py

@@ -609,7 +609,7 @@ class GridMixin(_RequestProto):
 class SearchMixin(_RequestProto):
     """Natural-language catalog search via POST /search."""
 
-    async def text_search_async(self, query: str) -> Dict[str, Any]:
+    async def search_async(self, query: str) -> Dict[str, Any]:
         """Search the DataQuery catalog using a natural-language query.
 
         POSTs ``{"query": query}`` to ``/search`` and returns the parsed JSON body.

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/core/client.py

@@ -181,7 +181,7 @@ class DataQueryClient(
             raise ConfigurationError("Invalid base_url format")
 
         if strict_oauth_check and self.config.oauth_enabled:
-            if not self.config.client_id or not self.config.client_secret:
+            if not self.config.client_id or not self.config.get_client_secret():
                 raise ConfigurationError("client_id and client_secret are required")
 
         if not self.auth_manager.is_authenticated():
@@ -494,33 +494,28 @@ class DataQueryClient(
         if self.session is None:
             raise NetworkError("Failed to establish connection")
 
-        try:
-            response = await self.session.request(method, url, **kwargs)
-
-            # Raise 429 inside the retry scope so the retry manager backs off and retries.
-            if response.status == 429:
-                self.rate_limiter.handle_rate_limit_response(dict(response.headers))
-                retry_after = self._parse_retry_after(response.headers)
-                # Drain body so the connection can be reused; ignore decode errors.
-                try:
-                    await response.text()
-                except (UnicodeDecodeError, aiohttp.ClientPayloadError):
-                    pass
-                raise RateLimitError(f"Rate limit exceeded: {response.status}", retry_after=retry_after)
-
-            if response.status >= 500:
-                # Drain body so the connection can be reused; ignore decode errors.
-                try:
-                    await response.text()
-                except (UnicodeDecodeError, aiohttp.ClientPayloadError):
-                    pass
-                raise NetworkError(f"Server error: {response.status}", status_code=response.status)
+        response = await self.session.request(method, url, **kwargs)
 
-            return response
-        except (NetworkError, RateLimitError):
-            raise
-        except Exception:
-            raise
+        # Raise 429 inside the retry scope so the retry manager backs off and retries.
+        if response.status == 429:
+            self.rate_limiter.handle_rate_limit_response(dict(response.headers))
+            retry_after = self._parse_retry_after(response.headers)
+            # Drain body so the connection can be reused; ignore decode errors.
+            try:
+                await response.text()
+            except (UnicodeDecodeError, aiohttp.ClientPayloadError):
+                pass
+            raise RateLimitError(f"Rate limit exceeded: {response.status}", retry_after=retry_after)
+
+        if response.status >= 500:
+            # Drain body so the connection can be reused; ignore decode errors.
+            try:
+                await response.text()
+            except (UnicodeDecodeError, aiohttp.ClientPayloadError):
+                pass
+            raise NetworkError(f"Server error: {response.status}", status_code=response.status)
+
+        return response
 
     async def list_groups_async(self, limit: Optional[int] = None) -> List[Group]:
         """

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/dataquery.py

@@ -10,12 +10,14 @@ import time
 from calendar import monthrange
 from datetime import date, datetime
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Tuple, Union
 
 import structlog
 from dotenv import load_dotenv
+from pydantic import SecretStr
 
 from .config import EnvConfig
+from .constants.download import NO_FILES_FOUND_ERROR
 from .core._sync import SyncRunner
 from .core.client import DataQueryClient
 from .types.exceptions import ConfigurationError
@@ -186,7 +188,7 @@ class DataQuery:
             if client_id:
                 self.client_config.client_id = client_id
             if client_secret:
-                self.client_config.client_secret = client_secret
+                self.client_config.client_secret = SecretStr(client_secret)
             if not self.client_config.oauth_token_url and self.client_config.base_url:
                 self.client_config.oauth_token_url = f"{self.client_config.base_url.rstrip('/')}/oauth/token"
 
@@ -717,7 +719,7 @@ class DataQuery:
         client = self._ensure_client()
         return await client.get_grid_data_async(expr, grid_id, date)
 
-    async def text_search_async(self, query: str) -> Dict[str, Any]:
+    async def search_async(self, query: str) -> Dict[str, Any]:
         """Search the DataQuery catalog using a natural-language query.
 
         Args:
@@ -728,7 +730,7 @@ class DataQuery:
         """
         await self.connect_async()
         client = self._ensure_client()
-        return await client.text_search_async(query)
+        return await client.search_async(query)
 
     async def run_groups_async(self, max_concurrent: int = 5) -> OperationReport:
         """Run complete operation for listing all groups."""
@@ -907,6 +909,7 @@ class DataQuery:
         delay_between_downloads: float = 0.2,
         max_retries: int = 3,
         file_group_id: Optional[Union[str, List[str]]] = None,
+        on_file_complete: Optional[Callable[["DownloadResult"], Awaitable[None]]] = None,
     ) -> OperationReport:
         """
         Download all files in a group for a date range using parallel HTTP range requests.
@@ -929,6 +932,11 @@ class DataQuery:
             file_group_id: Optional restriction to specific file-group-id(s). Accepts a
                 single id or a list of ids. When a list is supplied, availability is
                 queried in parallel per id and the union of dates is downloaded.
+            on_file_complete: Optional async callback awaited for each file as soon as
+                it finishes downloading (status ``completed``/``already_exists``). Runs
+                concurrently with the downloads still in flight, so post-processing such
+                as unzipping overlaps with subsequent downloads rather than waiting for
+                the whole batch.
 
         Returns:
             Dictionary with download results and statistics
@@ -1006,7 +1014,7 @@ class DataQuery:
                 total_time_minutes = total_time_seconds / 60.0
 
                 logger.warning(
-                    "No available files found for date range",
+                    NO_FILES_FOUND_ERROR,
                     group_id=group_id,
                     start_date=start_date,
                     end_date=end_date,
@@ -1014,7 +1022,7 @@ class DataQuery:
                 return OperationReport(
                     operation="group_download",
                     status="error",
-                    error="No available files found for date range",
+                    error=NO_FILES_FOUND_ERROR,
                     subject={"group_id": group_id, "start_date": start_date, "end_date": end_date},
                     counts={"total_files": 0, "successful_downloads": 0, "failed_downloads": 0},
                     timing={
@@ -1071,6 +1079,7 @@ class DataQuery:
                 base_retry_delay=delay_between_downloads,
                 max_retries=max_retries,
                 progress_callback=progress_callback,
+                on_file_complete=on_file_complete,
             )
 
             operation_end_time = time.time()
@@ -2047,9 +2056,9 @@ class DataQuery:
         """
         return self._run_sync(self.get_grid_data_async(expr, grid_id, date))
 
-    def text_search(self, query: str) -> Dict[str, Any]:
-        """Synchronous wrapper for :meth:`text_search_async`."""
-        return self._run_sync(self.text_search_async(query))
+    def search(self, query: str) -> Dict[str, Any]:
+        """Synchronous wrapper for :meth:`search_async`."""
+        return self._run_sync(self.search_async(query))
 
     def run_groups(self, max_concurrent: int = 5) -> OperationReport:
         """Synchronous wrapper for run_groups_async."""

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/download/parallel.py

@@ -16,7 +16,7 @@ import time
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import IO, TYPE_CHECKING, Callable, Optional
+from typing import IO, TYPE_CHECKING, Awaitable, Callable, Optional
 
 import aiohttp
 import structlog
@@ -530,6 +530,7 @@ async def _download_one_with_stagger(
     global_semaphore: asyncio.Semaphore,
     delay_seconds: float,
     progress_callback: Optional[Callable],
+    on_file_complete: Optional[Callable[[DownloadResult], Awaitable[None]]] = None,
 ) -> Optional[DownloadResult]:
     file_group_id = _file_id(file_info)
     file_datetime = _file_dt(file_info)
@@ -554,6 +555,21 @@ async def _download_one_with_stagger(
             file_datetime=file_datetime,
             status=result.status.value if result else "failed",
         )
+        if (
+            on_file_complete is not None
+            and result is not None
+            and result.status.value in ("completed", "already_exists")
+        ):
+            # Post-processing (e.g. unzip) runs here so it overlaps with the
+            # downloads of other files still in flight under the gather().
+            try:
+                await on_file_complete(result)
+            except Exception as cb_err:  # pragma: no cover - defensive
+                logger.warning(
+                    "on_file_complete callback failed",
+                    file_group_id=file_group_id,
+                    error=str(cb_err),
+                )
         return result
     except Exception as e:
         logger.error(
@@ -596,6 +612,7 @@ async def download_files_with_retry(
     base_retry_delay: float,
     max_retries: int,
     progress_callback: Optional[Callable] = None,
+    on_file_complete: Optional[Callable[[DownloadResult], Awaitable[None]]] = None,
 ) -> tuple[list[DownloadResult], list[dict], int]:
     """Run a staggered, retrying batch of parallel-range downloads.
 
@@ -605,6 +622,10 @@ async def download_files_with_retry(
     spread the burst against the API rate limiter. Failures are retried up
     to ``max_retries`` times with exponential backoff (``base_retry_delay``).
 
+    When ``on_file_complete`` is provided it is awaited for each successfully
+    downloaded file as soon as that file finishes, so post-processing (e.g.
+    unzipping) overlaps with the downloads of files still in flight.
+
     Returns ``(successful, failed, retry_count)``.
     """
 
@@ -618,6 +639,7 @@ async def download_files_with_retry(
                 global_semaphore=global_semaphore,
                 delay_seconds=i * intelligent_delay,
                 progress_callback=progress_callback,
+                on_file_complete=on_file_complete,
             )
             for i, fi in enumerate(batch)
         ]

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/sse/client.py

@@ -111,6 +111,7 @@ class SSEClient:
         params: Optional[dict] = None,
         event_id_store: Optional[SSEEventIdStore] = None,
         heartbeat_timeout: float = 0.0,
+        defer_event_id_persistence: bool = False,
     ):
         """
         Initialise the SSE client.
@@ -147,6 +148,16 @@ class SSEClient:
                     larger than the server's keep-alive interval. ``0`` (the
                     default) disables the watchdog and relies on the server
                     closing the stream cleanly.
+            defer_event_id_persistence: When ``True``, do not persist parsed
+                    event ids to ``event_id_store`` as they arrive. The owner
+                    (e.g. :class:`NotificationDownloadManager`) instead commits
+                    a low-water-mark only after the matching download has
+                    settled, so a crash mid-download replays the event instead
+                    of skipping it (at-least-once). ``event_id_store`` is still
+                    read once on construction to seed the first
+                    ``Last-Event-ID``. Defaults to ``False`` — persist at parse
+                    time, which is correct for standalone use where there is no
+                    download step to gate persistence on.
         """
         self.config = config
         self.auth_manager = auth_manager
@@ -158,6 +169,7 @@ class SSEClient:
         self.params = dict(params) if params else None
         self.event_id_store = event_id_store
         self.heartbeat_timeout = heartbeat_timeout
+        self._defer_event_id_persistence = defer_event_id_persistence
 
         self._running = False
         self._task: Optional[asyncio.Task] = None
@@ -481,7 +493,8 @@ class SSEClient:
                 event_id = field_value
                 if field_value and field_value.isdigit():
                     self._last_event_id = field_value
-                    self._persist_event_id(field_value)
+                    if not self._defer_event_id_persistence:
+                        self._persist_event_id(field_value)
             elif field_name == "retry":
                 try:
                     retry_ms = int(field_value)

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/sse/subscriber.py

@@ -6,8 +6,10 @@ downloads files as notifications arrive. Each SSE event uses the standard
 SSE format with the event type in the ``event:`` field (e.g., ``file-updated``)
 and the file details in the JSON ``data:`` payload.
 
-The SSE ``id:`` field is persisted to disk so that cross-process replay via
-the ``last-event-id`` query parameter works correctly on reconnection.
+The SSE ``id:`` field is persisted to disk as a low-water-mark — advanced past
+an event only once its download has settled — so cross-process replay via the
+``last-event-id`` query parameter re-delivers any file whose download did not
+complete before a restart (at-least-once delivery).
 """
 
 import asyncio
@@ -245,6 +247,13 @@ class NotificationDownloadManager:
         self._download_semaphore: Optional[asyncio.Semaphore] = None
         self._inflight: Set[asyncio.Task] = set()
 
+        # Replay low-water-mark. The persisted last-event-id is only advanced
+        # past an event once its download has settled, so a crash mid-download
+        # replays that event on restart instead of silently skipping it.
+        self._uncommitted_ids: Set[int] = set()
+        self._highest_seen_event_id: int = 0
+        self._committed_event_id: int = 0
+
         self.stats: Dict[str, Any] = {
             "start_time": None,
             "notifications_received": 0,
@@ -285,6 +294,12 @@ class NotificationDownloadManager:
             if self._event_id_store is not None:
                 stored_event_id = self._event_id_store.load()
 
+        if stored_event_id is not None and stored_event_id.isdigit():
+            # Seed the low-water-mark so we neither re-persist the stored id nor
+            # ever regress below it as newly received events settle.
+            self._committed_event_id = int(stored_event_id)
+            self._highest_seen_event_id = int(stored_event_id)
+
         if stored_event_id is not None:
             logger.info(
                 "Event replay enabled — resuming from event-id %s; skipping initial bulk check.",
@@ -307,6 +322,9 @@ class NotificationDownloadManager:
             params=self.subscription.query_params(),
             event_id_store=self._event_id_store,
             heartbeat_timeout=self._heartbeat_timeout,
+            # This manager commits a low-water-mark after downloads settle,
+            # so the client must not persist ids itself at parse time.
+            defer_event_id_persistence=True,
         )
         await self._sse_client.start()
 
@@ -393,6 +411,7 @@ class NotificationDownloadManager:
         if not self._running:
             return
 
+        self._register_event_id(event)
         task = asyncio.create_task(self._run_handler(event), name="sse-download")
         self._inflight.add(task)
         task.add_done_callback(self._inflight.discard)
@@ -407,6 +426,63 @@ class NotificationDownloadManager:
             except Exception as exc:
                 logger.error("Error handling notification: %s", exc)
                 await self._dispatch_error(exc)
+        # The download has settled (success, skip, or handled failure); advance
+        # the replay cursor. Deliberately not in a ``finally``: if the task is
+        # cancelled mid-download the cursor stays behind this event so it is
+        # replayed on restart rather than skipped.
+        await self._commit_settled_event(event)
+
+    def _register_event_id(self, event: SSEEvent) -> None:
+        """Record a numeric event id as in-flight before its download starts.
+
+        The replay cursor is never advanced past an in-flight id, so a crash
+        before the download settles leaves the event to be replayed.
+        """
+        if self._event_id_store is None:
+            return
+        eid = event.id
+        if not eid or not eid.isdigit():
+            return
+        value = int(eid)
+        self._uncommitted_ids.add(value)
+        if value > self._highest_seen_event_id:
+            self._highest_seen_event_id = value
+
+    def _next_commit_id(self) -> Optional[str]:
+        """Return the highest replay id now safe to persist, or ``None``.
+
+        The safe id is the low-water-mark: everything below the oldest
+        still-in-flight event has settled, so commit ``min(in-flight) - 1``
+        (or the highest id seen when nothing is in flight). Fully synchronous
+        so two concurrent settlers can't persist a decreasing watermark across
+        the ``await`` in :meth:`_commit_settled_event`.
+        """
+        if self._uncommitted_ids:
+            watermark = min(self._uncommitted_ids) - 1
+        else:
+            watermark = self._highest_seen_event_id
+        if watermark > self._committed_event_id:
+            self._committed_event_id = watermark
+            return str(watermark)
+        return None
+
+    async def _commit_settled_event(self, event: SSEEvent) -> None:
+        """Advance the persisted replay cursor after an event's handler settles.
+
+        Runs whether the download succeeded, was skipped, or failed — the id is
+        no longer in-flight either way. A terminal download failure is already
+        surfaced via ``error_callback``/``stats``; the cursor still advances so a
+        genuinely unfetchable file can't stall replay forever.
+        """
+        store = self._event_id_store
+        if store is None:
+            return
+        eid = event.id
+        if eid and eid.isdigit():
+            self._uncommitted_ids.discard(int(eid))
+        to_save = self._next_commit_id()
+        if to_save is not None:
+            await store.save(to_save)
 
     async def _on_sse_error(self, exc: Exception) -> None:
         """Called by SSEClient on connection errors."""

{dataquery_sdk-1.2.0 → dataquery_sdk-1.2.2}/dataquery/transport/auth.py

@@ -72,7 +72,7 @@ class TokenManager:
             Bearer token string or None if no valid token available
         """
         if self.config.has_bearer_token:
-            return f"Bearer {self.config.bearer_token}"
+            return f"Bearer {self.config.get_bearer_token()}"
 
         if not self.config.has_oauth_credentials:
             logger.warning("No OAuth credentials or bearer token configured")
@@ -111,7 +111,7 @@ class TokenManager:
         if not self.config.oauth_token_url:
             raise ConfigurationError("OAuth token URL not configured")
 
-        if not self.config.client_id or not self.config.client_secret:
+        if not self.config.client_id or not self.config.get_client_secret():
             raise ConfigurationError("client_id and client_secret are required for OAuth")
 
         token_request = TokenRequest(
@@ -172,7 +172,7 @@ class TokenManager:
                 "grant_type": "refresh_token",
                 "refresh_token": self.current_token.refresh_token,
                 "client_id": self.config.client_id,
-                "client_secret": self.config.client_secret,
+                "client_secret": self.config.get_client_secret(),
             }
 
             if not self.config.oauth_token_url: