snowloader 0.2.2__tar.gz → 0.2.4__tar.gz

{snowloader-0.2.2 → snowloader-0.2.4}/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to snowloader are documented here. This project follows [Semantic Versioning](https://semver.org/).
 
+## [0.2.4] - 2026-04-28
+
+### Changed
+
+- Truncated or malformed JSON responses are now treated as transient failures and retried up to `max_retries` instead of raising immediately. Some ServiceNow front ends occasionally return cut-off response bodies under sustained concurrent load; this change keeps long extractions alive through those blips.
+
+## [0.2.3] - 2026-04-28
+
+### Changed
+
+- `AsyncSnowConnection` now uses `force_close=True` and `limit_per_host=concurrency` on its `aiohttp.TCPConnector`. Each request gets a fresh TCP connection, which avoids the connection-reuse failures some ServiceNow instances exhibit under sustained concurrent load.
+- Non-object JSON bodies (e.g. `null` or a list returned with HTTP 200) are now treated as transient failures: the SDK retries up to `max_retries` and raises `SnowConnectionError` if the issue persists. Previously the v0.2.2 fallback silently treated them as empty pages, which would lose data.
+- HTTP 500 added to the default retryable status code set on both `SnowConnection` and `AsyncSnowConnection`. ServiceNow 500s are typically transient overload, not deterministic bugs.
+
+### Fixed
+
+- `AsyncSnowConnection.aget_records` no longer crashes with `AttributeError` when a page returns a non-object JSON body. The retry-then-raise behavior surfaces the failure clearly instead of silently dropping data.
+
 ## [0.2.2] - 2026-04-28
 
 ### Fixed

{snowloader-0.2.2 → snowloader-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowloader
-Version: 0.2.2
+Version: 0.2.4
 Summary: Comprehensive ServiceNow data loader for AI/LLM pipelines - Incidents, CMDB, KB, Changes, Catalog & more. Works with LangChain & LlamaIndex.
 Project-URL: Homepage, https://github.com/ronidas39/snowloader
 Project-URL: Documentation, https://snowloader.readthedocs.io

{snowloader-0.2.2 → snowloader-0.2.4}/docs/conf.py

@@ -10,7 +10,7 @@ sys.path.insert(0, os.path.abspath("../src"))
 project = "snowloader"
 copyright = "2026, Roni Das"
 author = "Roni Das"
-release = "0.2.2"
+release = "0.2.4"
 
 # -- General configuration ---------------------------------------------------
 

{snowloader-0.2.2 → snowloader-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "snowloader"
-version = "0.2.2"
+version = "0.2.4"
 description = "Comprehensive ServiceNow data loader for AI/LLM pipelines - Incidents, CMDB, KB, Changes, Catalog & more. Works with LangChain & LlamaIndex."
 readme = "README.md"
 license = {text = "MIT"}

{snowloader-0.2.2 → snowloader-0.2.4}/src/snowloader/__init__.py

@@ -21,7 +21,7 @@ from snowloader.loaders.problems import ProblemLoader
 from snowloader.models import BaseSnowLoader, SnowDocument
 from snowloader.utils.parsing import parse_labelled_int
 
-__version__ = "0.2.2"
+__version__ = "0.2.4"
 
 try:
     from snowloader.async_connection import AsyncSnowConnection  # noqa: F401

{snowloader-0.2.2 → snowloader-0.2.4}/src/snowloader/async_connection.py

@@ -42,7 +42,7 @@ logger = logging.getLogger(__name__)
 
 _DEFAULT_MAX_RETRIES = 3
 _DEFAULT_RETRY_BACKOFF = 1.0
-_RETRYABLE_STATUS_CODES = {429, 502, 503, 504}
+_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
 _MAX_PAGE_SIZE = 10000
 _MIN_PAGE_SIZE = 1
 _DEFAULT_CONCURRENCY = 16
@@ -210,9 +210,18 @@ class AsyncSnowConnection:
 
     async def _ensure_session(self) -> aiohttp.ClientSession:
         if self._session is None or self._session.closed:
+            # force_close=True disables HTTP keep-alive: each request gets a
+            # fresh TCP connection. Some ServiceNow front ends (and shared
+            # WAF/proxy layers in front of them) silently return empty / null
+            # response bodies on reused connections under concurrent load,
+            # which corrupts paginated reads. Trading a small amount of
+            # connection-setup overhead for correctness is the right call
+            # for a data-extraction SDK where missing pages are unacceptable.
             connector = aiohttp.TCPConnector(
                 limit=self.concurrency * 2,
+                limit_per_host=self.concurrency,
                 ssl=self._verify_ssl,
+                force_close=True,
             )
             timeout_cfg = aiohttp.ClientTimeout(total=self.timeout)
             auth = None
@@ -378,25 +387,57 @@ class AsyncSnowConnection:
                     try:
                         parsed = await resp.json(content_type=None)
                     except (aiohttp.ContentTypeError, ValueError) as exc:
+                        # Truncated or malformed JSON happens occasionally
+                        # under sustained concurrent load (server-side write
+                        # gets cut off). Retry like a transient failure.
+                        if attempt < self.max_retries:
+                            logger.warning(
+                                "API returned non-JSON / truncated response "
+                                "for %s %s: %s; retrying (attempt %d/%d)",
+                                method,
+                                url,
+                                exc,
+                                attempt + 1,
+                                self.max_retries,
+                            )
+                            await asyncio.sleep(backoff)
+                            attempt += 1
+                            backoff *= 2
+                            continue
                         raise SnowConnectionError(
-                            f"API returned non-JSON response for {method} {url}",
+                            f"API returned non-JSON response for {method} {url} "
+                            f"after {self.max_retries} retries",
                             status_code=resp.status,
                             detail=last_body,
                         ) from exc
 
                     if not isinstance(parsed, dict):
-                        # Some ServiceNow paths can return ``null`` or a list
-                        # under transient load. Treat anything that is not a
-                        # JSON object as an empty result so downstream code
-                        # does not crash on ``data.get(...)``.
-                        logger.warning(
-                            "API returned non-object JSON for %s %s (type=%s); "
-                            "treating as empty result.",
-                            method,
-                            url,
-                            type(parsed).__name__,
+                        # ServiceNow occasionally returns ``null`` or a list
+                        # body on a 200 response when something upstream
+                        # drops the payload (typically a connection-reuse
+                        # issue under concurrent load). Silently treating
+                        # this as an empty page would lose data, so retry
+                        # the request as if it were a transient 5xx.
+                        if attempt < self.max_retries:
+                            logger.warning(
+                                "API returned non-object JSON for %s %s (type=%s); "
+                                "retrying (attempt %d/%d)",
+                                method,
+                                url,
+                                type(parsed).__name__,
+                                attempt + 1,
+                                self.max_retries,
+                            )
+                            await asyncio.sleep(backoff)
+                            attempt += 1
+                            backoff *= 2
+                            continue
+                        raise SnowConnectionError(
+                            f"API returned non-object JSON for {method} {url} "
+                            f"after {self.max_retries} retries",
+                            status_code=resp.status,
+                            detail=f"Got {type(parsed).__name__} instead of dict",
                         )
-                        return {"result": []}
                     return cast(dict[str, Any], parsed)
             except aiohttp.ClientError as exc:
                 if attempt < self.max_retries:

{snowloader-0.2.2 → snowloader-0.2.4}/src/snowloader/connection.py

@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 # Retry defaults
 _DEFAULT_MAX_RETRIES = 3
 _DEFAULT_RETRY_BACKOFF = 1.0  # seconds, doubles each attempt
-_RETRYABLE_STATUS_CODES = {429, 502, 503, 504}
+_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
 _MAX_PAGE_SIZE = 10000
 _MIN_PAGE_SIZE = 1
 

{snowloader-0.2.2 → snowloader-0.2.4}/tests/unit/test_async_connection.py

@@ -255,12 +255,8 @@ async def test_aget_attachment_returns_bytes() -> None:
 
 
 @pytest.mark.asyncio
-async def test_request_treats_null_body_as_empty_result() -> None:
-    """Regression: ServiceNow can return JSON null under transient load.
-
-    Previously this crashed aget_records on ``data.get('result')``. Now we
-    log a warning and treat it as an empty result.
-    """
+async def test_request_retries_malformed_json_then_raises() -> None:
+    """Truncated or malformed JSON triggers retry, then raises after max_retries."""
     with aioresponses() as m:
         m.get(
             _stats_url(),
@@ -270,7 +266,7 @@ async def test_request_treats_null_body_as_empty_result() -> None:
         )
         m.get(
             _table_url(),
-            body="null",
+            body='{"result": [{"sys_id": "abc"',  # truncated mid-string
             status=200,
             content_type="application/json",
             repeat=True,
@@ -280,14 +276,23 @@ async def test_request_treats_null_body_as_empty_result() -> None:
             username="u",
             password="p",
             page_size=10,
+            max_retries=2,
+            retry_backoff=0.0,
         ) as conn:
-            records = [rec async for rec in conn.aget_records("incident")]
-            assert records == []
+            with pytest.raises(SnowConnectionError) as exc_info:
+                async for _ in conn.aget_records("incident"):
+                    pass
+            assert "non-JSON" in str(exc_info.value)
 
 
 @pytest.mark.asyncio
-async def test_request_treats_list_body_as_empty_result() -> None:
-    """Regression: defensive handling for unexpected non-object JSON shapes."""
+async def test_request_retries_null_body_then_raises() -> None:
+    """ServiceNow occasionally returns 200 with a ``null`` body under load.
+
+    The SDK must NOT silently treat this as an empty page (that would lose
+    data). Instead it retries up to ``max_retries`` and raises if the issue
+    persists, so the caller knows something is wrong.
+    """
     with aioresponses() as m:
         m.get(
             _stats_url(),
@@ -297,18 +302,61 @@ async def test_request_treats_list_body_as_empty_result() -> None:
         )
         m.get(
             _table_url(),
-            payload=[1, 2, 3],
+            body="null",
+            status=200,
+            content_type="application/json",
+            repeat=True,
+        )
+        async with AsyncSnowConnection(
+            instance_url=INSTANCE,
+            username="u",
+            password="p",
+            page_size=10,
+            max_retries=2,
+            retry_backoff=0.0,
+        ) as conn:
+            with pytest.raises(SnowConnectionError) as exc_info:
+                async for _ in conn.aget_records("incident"):
+                    pass
+            assert "non-object JSON" in str(exc_info.value)
+
+
+@pytest.mark.asyncio
+async def test_request_recovers_when_null_body_then_valid() -> None:
+    """When a transient null body is followed by a valid response, the SDK
+    must recover (return the valid data) rather than fail or skip."""
+    payload = {"result": [{"sys_id": "abc", "number": "INC1"}]}
+    call_count = {"n": 0}
+
+    def callback(url: str, **kwargs: object) -> object:
+        call_count["n"] += 1
+        if call_count["n"] == 1:
+            from aioresponses.core import CallbackResult
+
+            return CallbackResult(status=200, body="null", content_type="application/json")
+        from aioresponses.core import CallbackResult
+
+        return CallbackResult(status=200, payload=payload)
+
+    with aioresponses() as m:
+        m.get(
+            _stats_url(),
+            payload={"result": {"stats": {"count": "1"}}},
             status=200,
             repeat=True,
         )
+        m.get(_table_url(), callback=callback, repeat=True)
         async with AsyncSnowConnection(
             instance_url=INSTANCE,
             username="u",
             password="p",
             page_size=10,
+            max_retries=3,
+            retry_backoff=0.0,
         ) as conn:
             records = [rec async for rec in conn.aget_records("incident")]
-            assert records == []
+            assert len(records) == 1
+            assert records[0]["sys_id"] == "abc"
 
 
 @pytest.mark.asyncio