socketsecurity 2.6.0__tar.gz → 2.6.1__tar.gz

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2.6.1
+
+### Changed: scan comparison now polls the diff-scans endpoints
+
+- Diff mode no longer holds a single idle HTTP connection open while the API
+  computes the scan comparison. The CLI now creates a diff-scan resource
+  (`POST /orgs/{org}/diff-scans/from-ids`) and polls
+  `GET /orgs/{org}/diff-scans/{id}?cached=true` with short, bounded requests
+  until the comparison is ready (HTTP 200 instead of 202). This fixes
+  intermittent `Connection reset by peer` failures on the final comparison
+  step when scans take several minutes to compare and network middleboxes
+  (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout)
+  reap the idle connection.
+- Duplicate scan pairs are resolved after an HTTP 409 and then polled through
+  the same cached endpoint. This avoids automatically following the API's 302
+  duplicate redirect with an uncached, potentially long-lived GET request.
+- The change is transparent: no flags or workflow changes are needed. If the
+  org API token is missing the `diff-scans:create`, `diff-scans:list` or
+  `full-scans:list` scopes — or the new flow fails for any other reason — the
+  CLI logs a warning and falls back to the legacy streaming comparison.
+- Requires the pinned `socketdev==3.5.0` SDK.
+
 ## 2.6.0
 
 ### Changed: pin all Python dependencies

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: socketsecurity
-Version: 2.6.0
+Version: 2.6.1
 Summary: Socket Security CLI for CI/CD
 Project-URL: Homepage, https://socket.dev
 Author-email: Douglas Coburn <douglas@socket.dev>

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.6.0"
+version = "2.6.1"
 requires-python = ">= 3.11"
 license = {"file" = "LICENSE"}
 dependencies = [

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/socketsecurity/__init__.py

@@ -1,3 +1,3 @@
 __author__ = 'socket.dev'
-__version__ = '2.6.0'
+__version__ = '2.6.1'
 USER_AGENT = f'SocketPythonCLI/{__version__}'

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/socketsecurity/core/__init__.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
     from socketsecurity.config import CliConfig
 from socketdev import socketdev
 from socketdev.exceptions import APIFailure
-from socketdev.fullscans import FullScanParams, SocketArtifact
+from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact
 from socketdev.org import Organization
 from socketdev.repos import RepositoryInfo
 import copy
@@ -92,6 +92,25 @@ FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS = (10.0, 30.0, None)
 FULL_SCAN_UPLOAD_MAX_ATTEMPTS = len(FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS)
 FULL_SCAN_UPLOAD_BACKOFF_JITTER_SECONDS = 2.0
 
+# Diff-scan polling policy. The legacy scan comparison (fullscans.stream_diff) holds a
+# single HTTP connection open, fully idle, while the backend computes the diff; network
+# middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to
+# 4 minutes) kill that connection with a RST, surfacing as an intermittent
+# ConnectionResetError on large scans. The diff-scans flow instead creates a
+# diff-scan resource and polls its cached endpoint with short bounded requests: the API
+# answers 202 while the comparison is still computing and 200 with the result once it is
+# ready, so no connection is ever idle long enough to be reaped.
+#
+# Each poll consumes 1 unit of API quota, so the interval backs off toward
+# DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take
+# minutes to compute. The timeout is a backstop against a diff scan that never
+# completes; on expiry (or any other failure of this flow) the caller falls back to the
+# legacy streaming comparison rather than failing the scan outright.
+DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0
+DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0
+DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5
+DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0
+
 
 def _humanize_alert_type(alert_type: str) -> str:
     """Convert a camelCase/PascalCase alert type into a Title-Cased label.
@@ -1303,6 +1322,120 @@ class Core:
         
         return packages
 
+    def get_diff_scan_artifacts(
+            self,
+            head_full_scan_id: str,
+            new_full_scan_id: str
+    ) -> DiffArtifacts:
+        """Compare two full scans via the diff-scans endpoints, polling for the result.
+
+        Creates a diff-scan resource from the two full scan IDs, then polls
+        ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the
+        computed comparison (200) instead of a processing status (202). Unlike the
+        legacy ``fullscans.stream_diff`` call, no request is ever left idle while
+        the backend computes, so the comparison survives network idle timeouts.
+        See the DIFF_SCAN_POLL_* constants for the polling policy.
+
+        Requires an org token with the ``diff-scans:create``, ``diff-scans:list``
+        and ``full-scans:list`` scopes; callers are expected to catch failures and
+        fall back to the legacy streaming comparison.
+
+        Note that cached diff-scan responses always embed per-package license
+        details (the API ignores ``omit_license_details`` when ``cached=true``),
+        so unlike the legacy streaming comparison there is no lean-response
+        option here; see the comment on ``poll_params`` below.
+
+        Args:
+            head_full_scan_id: The before/base full scan ID
+            new_full_scan_id: The after/head full scan ID
+
+        Returns:
+            DiffArtifacts with the added/removed/unchanged/replaced/updated lists
+        """
+        create_params = {
+            "before": head_full_scan_id,
+            "after": new_full_scan_id,
+            "description": f"Socket Security CLI v{__version__} scan comparison",
+        }
+        try:
+            result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params)
+            diff_scan = result.get("diff_scan") or {}
+            response_summary = result
+        except APIFailure as error:
+            if error.status_code != 409:
+                raise
+
+            # Do not use on_duplicate=redirect here. The SDK follows that 302
+            # automatically with a GET that lacks cached=true, which can leave
+            # the connection idle while an existing diff scan is still computing.
+            # Resolve the duplicate resource explicitly so every result fetch
+            # continues through the bounded cached polling path below.
+            existing = self.sdk.diffscans.list(
+                self.config.org_slug,
+                params={
+                    "before_full_scan_id": head_full_scan_id,
+                    "after_full_scan_id": new_full_scan_id,
+                    "per_page": 1,
+                },
+            )
+            matches = existing.get("results") or []
+            diff_scan = matches[0] if matches else {}
+            response_summary = existing
+
+        diff_scan_id = diff_scan.get("id")
+        if not diff_scan_id:
+            raise Exception(
+                "Error creating or resolving diff scan: "
+                f"unexpected response: {str(response_summary)[:500]}"
+            )
+        artifacts_dict = diff_scan.get("artifacts")
+
+        # cached=true is the polling contract (202 while computing, 200 when
+        # ready). The API ignores omit_license_details when cached=true - cached
+        # results always embed license details - so there is no lean-response
+        # option on this path (unlike stream_diff with
+        # include_license_details=false, the lean-payload mitigation). If that extra
+        # payload ever gets a response truncated on a huge dependency tree,
+        # response.json() fails and the caller falls back to the legacy
+        # streaming comparison, which still requests the lean payload.
+        poll_params = {"cached": "true"}
+        deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS
+        interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS
+        while artifacts_dict is None:
+            try:
+                response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params)
+            except APIFailure as error:
+                if not error.is_transient_error():
+                    raise
+                # A dropped/timed-out poll is retryable: the diff scan keeps
+                # computing server-side regardless of what happens to any one poll.
+                log.warning(
+                    f"Transient error polling diff scan {diff_scan_id} "
+                    f"({type(error).__name__}), retrying in {interval:.0f}s"
+                )
+                response = {"status": "processing"}
+            if response.get("status") != "processing":
+                scan = response.get("diff_scan") or {}
+                if scan.get("artifacts") is None:
+                    raise Exception(
+                        f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}"
+                    )
+                artifacts_dict = scan["artifacts"]
+                break
+            if time.monotonic() >= deadline:
+                raise Exception(
+                    f"Timed out waiting for diff scan {diff_scan_id} after "
+                    f"{DIFF_SCAN_POLL_TIMEOUT_SECONDS:.0f} seconds"
+                )
+            log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s")
+            time.sleep(interval)
+            interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS)
+
+        return DiffArtifacts.from_dict({
+            key: artifacts_dict.get(key) or []
+            for key in ("added", "removed", "unchanged", "replaced", "updated")
+        })
+
     def get_added_and_removed_packages(
             self,
             head_full_scan_id: str,
@@ -1315,8 +1448,12 @@ class Core:
         Args:
             head_full_scan_id: Previous scan (maybe None if first scan)
             new_full_scan_id: New scan just created
-            include_license_details: Whether to ask the diff endpoint to embed
-                per-package license attribution/details in the response.
+            include_license_details: Whether to ask the *legacy streaming* diff
+                endpoint to embed per-package license attribution/details in the
+                response. Only consulted on the fallback path: the primary
+                diff-scans path always receives embedded license details, since
+                the API ignores ``omit_license_details`` for cached reads (see
+                get_diff_scan_artifacts).
 
                 Defaults to ``False`` on purpose. The diff endpoint exists to
                 compare alerts between two scans; the license fields it can embed
@@ -1343,39 +1480,55 @@ class Core:
 
         log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}")
         diff_start = time.time()
+        diff_artifacts = None
         try:
-            diff_report = (
-                self.sdk.fullscans.stream_diff(
-                    self.config.org_slug,
-                    head_full_scan_id,
-                    new_full_scan_id,
-                    use_types=True,
-                    include_license_details=str(include_license_details).lower()
-                ).data
+            diff_artifacts = self.get_diff_scan_artifacts(
+                head_full_scan_id,
+                new_full_scan_id
             )
-        except APIFailure as e:
-            log.error(f"API Error: {e}")
-            if self.cli_config and self.cli_config.disable_blocking:
-                sys.exit(0)
-            sys.exit(1)
-        except Exception as e:
-            import traceback
-            log.error(f"Error getting diff report: {str(e)}")
-            log.error(f"Stack trace:\n{traceback.format_exc()}")
-            raise
+        except Exception as error:
+            # SDK error messages can span many lines (path + response headers); the
+            # first line carries the status, which is all the warning needs.
+            error_summary = str(error).strip().splitlines()[0] if str(error).strip() else ""
+            log.warning(
+                f"Diff scan comparison failed with {type(error).__name__}({error_summary}), "
+                "falling back to the streaming scan comparison"
+            )
+
+        if diff_artifacts is None:
+            try:
+                diff_artifacts = (
+                    self.sdk.fullscans.stream_diff(
+                        self.config.org_slug,
+                        head_full_scan_id,
+                        new_full_scan_id,
+                        use_types=True,
+                        include_license_details=str(include_license_details).lower()
+                    ).data.artifacts
+                )
+            except APIFailure as e:
+                log.error(f"API Error: {e}")
+                if self.cli_config and self.cli_config.disable_blocking:
+                    sys.exit(0)
+                sys.exit(1)
+            except Exception as e:
+                import traceback
+                log.error(f"Error getting diff report: {str(e)}")
+                log.error(f"Stack trace:\n{traceback.format_exc()}")
+                raise
 
         diff_end = time.time()
         log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds")
         log.info("Diff report artifact counts:")
-        log.info(f"Added: {len(diff_report.artifacts.added)}")
-        log.info(f"Removed: {len(diff_report.artifacts.removed)}")
-        log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}")
-        log.info(f"Replaced: {len(diff_report.artifacts.replaced)}")
-        log.info(f"Updated: {len(diff_report.artifacts.updated)}")
-
-        added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated
-        removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced
-        unchanged_artifacts = diff_report.artifacts.unchanged
+        log.info(f"Added: {len(diff_artifacts.added)}")
+        log.info(f"Removed: {len(diff_artifacts.removed)}")
+        log.info(f"Unchanged: {len(diff_artifacts.unchanged)}")
+        log.info(f"Replaced: {len(diff_artifacts.replaced)}")
+        log.info(f"Updated: {len(diff_artifacts.updated)}")
+
+        added_artifacts = diff_artifacts.added + diff_artifacts.updated
+        removed_artifacts = diff_artifacts.removed + diff_artifacts.replaced
+        unchanged_artifacts = diff_artifacts.unchanged
 
         added_packages: Dict[str, Package] = {}
         removed_packages: Dict[str, Package] = {}

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/tests/core/conftest.py

@@ -87,6 +87,22 @@ def stream_diff_response(data_dir, load_json):
     })
 
 
+@pytest.fixture
+def diff_scan_get_response(data_dir, load_json):
+    """GET /orgs/{org}/diff-scans/{id} response built from the stream_diff fixture.
+
+    The diff-scans endpoint returns the same artifact shape as the legacy
+    streaming diff, wrapped in a diff_scan object.
+    """
+    json_data = load_json(data_dir / "fullscans" / "diff" / "stream_diff.json")
+    return {
+        "diff_scan": {
+            "id": "diff-scan-123",
+            "artifacts": json_data["data"]["artifacts"],
+        }
+    }
+
+
 
 
 
@@ -138,6 +154,7 @@ def mock_sdk_with_responses(
     new_scan_metadata,
     new_scan_stream,
     stream_diff_response,
+    diff_scan_get_response,
     create_full_scan_response,
 ):
     sdk = mock_socket_sdk.return_value
@@ -173,4 +190,8 @@ def mock_sdk_with_responses(
         lambda org_slug, head_id, new_id, **kwargs: stream_diff_response
     )
 
+    # Diff-scans endpoints (primary scan-comparison path)
+    sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-123"}}
+    sdk.diffscans.get.return_value = diff_scan_get_response
+
     return sdk

socketsecurity-2.6.1/tests/core/test_diff_scan_polling.py

@@ -0,0 +1,122 @@
+"""Tests for the diff-scans polling scan comparison.
+
+The comparison must never hold an idle connection open: it creates a diff-scan
+resource and polls the cached endpoint (202 while processing, 200 when ready),
+falling back to the legacy streaming diff if the new flow is unavailable.
+"""
+import pytest
+from socketdev.exceptions import APIConnectionError, APIFailure
+
+import socketsecurity.core as core_module
+from socketsecurity.core import Core
+from socketsecurity.core.socket_config import SocketConfig
+
+
+@pytest.fixture
+def core(mock_sdk_with_responses):
+    config = SocketConfig(api_key="test_key")
+    return Core(config=config, sdk=mock_sdk_with_responses)
+
+
+@pytest.fixture
+def no_sleep(mocker):
+    return mocker.patch("socketsecurity.core.time.sleep")
+
+
+def test_polls_until_diff_scan_ready(core, diff_scan_get_response, no_sleep):
+    """202 processing responses are polled through until the 200 result arrives."""
+    processing = {"status": "processing", "id": "diff-scan-123"}
+    core.sdk.diffscans.get.side_effect = [processing, processing, diff_scan_get_response]
+
+    artifacts = core.get_diff_scan_artifacts("head", "new")
+
+    assert core.sdk.diffscans.get.call_count == 3
+    assert no_sleep.call_count == 2  # slept between polls, never during them
+    assert len(artifacts.added) > 0
+
+
+def test_poll_interval_backs_off(core, diff_scan_get_response, no_sleep, monkeypatch):
+    """The poll interval grows toward the max so long comparisons stay quota-friendly."""
+    monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS", 4.0)
+    monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS", 10.0)
+    processing = {"status": "processing", "id": "diff-scan-123"}
+    core.sdk.diffscans.get.side_effect = [processing] * 4 + [diff_scan_get_response]
+
+    core.get_diff_scan_artifacts("head", "new")
+
+    waits = [call.args[0] for call in no_sleep.call_args_list]
+    assert waits == [4.0, 6.0, 9.0, 10.0]  # 1.5x backoff, capped at the max
+
+
+def test_transient_poll_error_is_retried(core, diff_scan_get_response, no_sleep):
+    """A dropped poll doesn't abandon the flow - the diff keeps computing server-side."""
+    core.sdk.diffscans.get.side_effect = [APIConnectionError("reset"), diff_scan_get_response]
+
+    artifacts = core.get_diff_scan_artifacts("head", "new")
+
+    assert core.sdk.diffscans.get.call_count == 2
+    assert len(artifacts.added) > 0
+
+
+def test_non_transient_poll_error_raises(core, no_sleep):
+    """Deterministic API errors (e.g. 403 missing scopes) propagate to the caller."""
+    core.sdk.diffscans.get.side_effect = APIFailure("forbidden", status_code=403)
+
+    with pytest.raises(APIFailure):
+        core.get_diff_scan_artifacts("head", "new")
+
+
+def test_poll_timeout_raises(core, no_sleep, monkeypatch):
+    """A diff scan that never completes hits the polling backstop."""
+    monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_TIMEOUT_SECONDS", 0.0)
+    core.sdk.diffscans.get.return_value = {"status": "processing", "id": "diff-scan-123"}
+
+    with pytest.raises(Exception, match="Timed out waiting for diff scan"):
+        core.get_diff_scan_artifacts("head", "new")
+
+
+def test_duplicate_conflict_uses_cached_polling(core, diff_scan_get_response):
+    """A duplicate is resolved explicitly so the SDK cannot follow an uncached redirect."""
+    core.sdk.diffscans.create_from_ids.side_effect = APIFailure(
+        "duplicate", status_code=409
+    )
+    core.sdk.diffscans.list.return_value = {
+        "results": [{"id": "existing-diff-scan"}],
+    }
+
+    artifacts = core.get_diff_scan_artifacts("head", "new")
+
+    create_params = core.sdk.diffscans.create_from_ids.call_args.args[1]
+    assert "on_duplicate" not in create_params
+    core.sdk.diffscans.list.assert_called_once_with(
+        core.config.org_slug,
+        params={
+            "before_full_scan_id": "head",
+            "after_full_scan_id": "new",
+            "per_page": 1,
+        },
+    )
+    core.sdk.diffscans.get.assert_called_once_with(
+        core.config.org_slug,
+        "existing-diff-scan",
+        params={"cached": "true"},
+    )
+    assert len(artifacts.added) > 0
+
+
+def test_fallback_to_streaming_diff_on_failure(core):
+    """If the diff-scans flow fails (e.g. token missing the diff-scans scopes),
+    the comparison falls back to the legacy streaming diff transparently."""
+    core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403)
+
+    added, removed, all_packages = core.get_added_and_removed_packages("head", "new")
+
+    core.sdk.fullscans.stream_diff.assert_called_once_with(
+        core.config.org_slug,
+        "head",
+        "new",
+        use_types=True,
+        include_license_details="false",
+    )
+    assert "dp3" in added
+    assert "dp2" in removed

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/tests/core/test_sdk_methods.py

@@ -225,19 +225,27 @@ def test_get_added_and_removed_packages(core):
     """Test getting added and removed packages between two scans"""
     # Get two different scans to compare
     added, removed, all_packages = core.get_added_and_removed_packages("head", "new")
-    
-    # Verify SDK was called correctly.
-    # include_license_details defaults to "false": the diff path never consumes
-    # embedded license data (license artifacts come from the PURL endpoint), so
-    # requesting it only bloats the response and risks the truncation
-    # crash on large repos.
-    core.sdk.fullscans.stream_diff.assert_called_once_with(
+
+    # Verify SDK was called correctly: the comparison goes through the diff-scans
+    # endpoints (create + poll) rather than the legacy streaming diff, so no
+    # connection is left idle while the backend computes.
+    create_args = core.sdk.diffscans.create_from_ids.call_args
+    assert create_args[0][0] == core.config.org_slug
+    create_params = create_args[0][1]
+    assert create_params["before"] == "head"
+    assert create_params["after"] == "new"
+    assert "on_duplicate" not in create_params
+
+    # cached=true is the polling contract (202 while computing, 200 when ready).
+    # No omit_license_details param: the API ignores it for cached reads (cached
+    # results always embed license details), so sending it would only suggest a
+    # leanness guarantee this path doesn't have.
+    core.sdk.diffscans.get.assert_called_once_with(
         core.config.org_slug,
-        "head",
-        "new",
-        use_types=True,
-        include_license_details="false",
+        "diff-scan-123",
+        params={"cached": "true"},
     )
+    core.sdk.fullscans.stream_diff.assert_not_called()
     
     # Verify the results
     # Added packages
@@ -252,7 +260,13 @@ def test_get_added_and_removed_packages(core):
     assert "pypi/direct_package_1@1.6.0" in all_packages  # Unchanged package is in full package map
 
 def test_get_added_and_removed_packages_license_override(core):
-    """The include_license_details override seam still works when explicitly requested."""
+    """include_license_details only governs the legacy fallback path now: the
+    diff-scans path always receives embedded license details (the API ignores
+    omit_license_details for cached reads), so the seam must survive through to
+    the stream_diff call when the primary path is unavailable."""
+    from socketdev.exceptions import APIFailure
+
+    core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403)
     core.get_added_and_removed_packages("head", "new", include_license_details=True)
 
     core.sdk.fullscans.stream_diff.assert_called_once_with(

{socketsecurity-2.6.0 → socketsecurity-2.6.1}/uv.lock

@@ -1282,7 +1282,7 @@ wheels = [
 
 [[package]]
 name = "socketsecurity"
-version = "2.6.0"
+version = "2.6.1"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },