port-ocean 0.30.2__py3-none-any.whl → 0.30.3__py3-none-any.whl

port_ocean/helpers/async_client.py

@@ -5,6 +5,7 @@ from loguru import logger
 
 from port_ocean.helpers.retry import RetryTransport, RetryConfig
 from port_ocean.helpers.stream import Stream
+from typing import AsyncGenerator, List, Dict
 
 
 class OceanAsyncClient(httpx.AsyncClient):
@@ -56,3 +57,31 @@ class OceanAsyncClient(httpx.AsyncClient):
         response = await self.send(req, stream=True)
         response.raise_for_status()
         return Stream(response)
+
+
+class StreamingClientWrapper:
+    def __init__(self, http_client: OceanAsyncClient):
+        self._client = http_client
+
+    async def stream_json(
+        self,
+        url: str,
+        target_items_path: str,
+        **kwargs: Any,
+    ) -> AsyncGenerator[List[Dict[str, Any]], None]:
+        """
+        A wrapper that provides a unified async generator interface for both streaming
+        and non-streaming HTTP GET requests.
+
+        :param url: The URL to request.
+        :param target_items_path: A JMESPath string to extract the list of items
+                                  from the JSON response (e.g., 'results'). The wrapper
+                                  will automatically adapt this for the streaming parser.
+        :param kwargs: Additional arguments for the HTTP request.
+        """
+        # ijson needs a path to the items inside the array, e.g., "results.item"
+        streaming_path = f"{target_items_path}.item"
+        stream_response = await self._client.get_stream(url, **kwargs)
+        json_stream = stream_response.get_json_stream(target_items=streaming_path)
+        async for items_batch in json_stream:
+            yield items_batch

port_ocean/helpers/stream.py

@@ -22,7 +22,10 @@ class Stream:
         if chunk_size is None:
             chunk_size = ocean_context.ocean.config.streaming.chunk_size
 
-        file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
+        streaming_location = ocean_context.ocean.config.streaming.location
+        os.makedirs(streaming_location, exist_ok=True)
+
+        file_name = f"{streaming_location}/{uuid.uuid4()}"
 
         crypt = Fernet(Fernet.generate_key())
 

port_ocean/tests/clients/test_streaming_wrapper.py

@@ -0,0 +1,85 @@
+import pytest
+from unittest.mock import AsyncMock
+from typing import Any, AsyncGenerator
+
+
+from port_ocean.helpers.async_client import OceanAsyncClient, StreamingClientWrapper
+from port_ocean.helpers.stream import Stream
+
+
+class MockStream:
+    """A mock of the port_ocean.helpers.stream.Stream class for testing."""
+
+    def __init__(self, data_chunks: list[list[dict[str, Any]]]) -> None:
+        self._data_chunks = data_chunks
+
+    async def get_json_stream(self, target_items: str) -> AsyncGenerator[Any, None]:
+        """An async generator that yields the data chunks."""
+        for chunk in self._data_chunks:
+            yield chunk
+
+
+@pytest.mark.asyncio
+async def test_stream_json_with_streaming_enabled() -> None:
+    """
+    Tests that when streaming is enabled, the wrapper uses the `get_stream` method
+    and yields data in batches.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    # Simulate receiving the data in two separate chunks/batches
+    mock_stream_instance = MockStream(
+        [[{"id": 1, "name": "one"}], [{"id": 2, "name": "two"}]]
+    )
+    mock_client.get_stream.return_value = mock_stream_instance
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    results = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    mock_client.get_stream.assert_called_once_with("http://test.com")
+    mock_client.get.assert_not_called()
+    assert results == [[{"id": 1, "name": "one"}], [{"id": 2, "name": "two"}]]
+
+
+@pytest.mark.asyncio
+async def test_stream_json_with_empty_results_streaming() -> None:
+    """
+    Tests that the wrapper handles an empty stream correctly in streaming mode.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    mock_stream_instance = MockStream([[]])  # Stream yields one empty batch
+    mock_client.get_stream.return_value = mock_stream_instance
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    results = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    assert results == [[]]
+
+
+@pytest.mark.asyncio
+async def test_stream_json_path_adaptation_for_streaming() -> None:
+    """
+    Tests that the wrapper correctly adapts the `target_items_path` for the
+    streaming parser (ijson) by appending `.item`.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    mock_stream = AsyncMock(spec=Stream)
+    mock_client.get_stream.return_value = mock_stream
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    # We only need to trigger the call to check the arguments
+    _ = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    # Verify that get_json_stream was called with the modified path
+    mock_stream.get_json_stream.assert_called_once_with(target_items="results.item")

{port_ocean-0.30.2.dist-info → port_ocean-0.30.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.30.2
+Version: 0.30.3
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -33,6 +33,7 @@ Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: ijson (>=3.4.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.6)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
+Requires-Dist: jmespath (>=1.0.1,<2.0.0)
 Requires-Dist: jq (>=1.8.0,<2.0.0)
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)

{port_ocean-0.30.2.dist-info → port_ocean-0.30.3.dist-info}/RECORD

@@ -147,11 +147,11 @@ port_ocean/exceptions/port_defaults.py,sha256=2a7Koy541KxMan33mU-gbauUxsumG3NT4i
 port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnqI,197
 port_ocean/exceptions/webhook_processor.py,sha256=4SnkVzVwiacH_Ip4qs1hRHa6GanhnojW_TLTdQQtm7Y,363
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/helpers/async_client.py,sha256=M8gKUjX8ZwRbmJ-U6KNq-p-nfGr0CwHdS0eN_pbZAJ0,2103
+port_ocean/helpers/async_client.py,sha256=GDRNq9kqb-7o6pAxSNZvE7zGh5semTr0oHoADuNjP-E,3325
 port_ocean/helpers/metric/metric.py,sha256=9pxpDT-nP0nTZ2kIutCTFuF7IwOQcSJ3dkBA1CVSurc,14626
 port_ocean/helpers/metric/utils.py,sha256=1lAgrxnZLuR_wUNDyPOPzLrm32b8cDdioob2lvnPQ1A,1619
 port_ocean/helpers/retry.py,sha256=UiOUo89hUrY0VVLL8sMR8GAYg-UVf2Y3yJ-8hBU1I7E,20285
-port_ocean/helpers/stream.py,sha256=_UwsThzXynxWzL8OlBT1pmb2evZBi9HaaqeAGNuTuOI,2338
+port_ocean/helpers/stream.py,sha256=i984XbY0hGWTuv7oe4k1byUkCd79D-HVBH3ijxsDgmM,2442
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/log/handlers.py,sha256=LJ1WAfq7wYCrBpeLPihMKmWjdSahKKXNHFMRYkbk0Co,3630
 port_ocean/log/logger_setup.py,sha256=wcr5WOkYRtng4pW6ZRl4Av3GqtZ2omSWIqYhB_8Duuc,2700
@@ -171,6 +171,7 @@ port_ocean/tests/clients/oauth/test_oauth_client.py,sha256=2XVMQUalDpiD539Z7_dk5
 port_ocean/tests/clients/port/mixins/test_entities.py,sha256=jOMJ3ICUlhjjPTo4q6qUrEjTKvXRLUE6KjqjdFiDRBY,10766
 port_ocean/tests/clients/port/mixins/test_integrations.py,sha256=vRt_EMsLozQC1LJNXxlvnHj3-FlOBGgAYxg5T0IAqtA,7621
 port_ocean/tests/clients/port/mixins/test_organization_mixin.py,sha256=zzKYz3h8dl4Z5A2QG_924m0y9U6XTth1XYOfwNrd_24,914
+port_ocean/tests/clients/test_streaming_wrapper.py,sha256=uTc0uaNynG7UgbRKNFaHURh8GYdG1O_cbS89veCxk-U,2902
 port_ocean/tests/config/test_config.py,sha256=Rk4N-ldVSOfn1p23NzdVdfqUpPrqG2cMut4Sv-sAOrw,1023
 port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,101
 port_ocean/tests/core/conftest.py,sha256=0Oql7R1iTbjPyNdUoO6M21IKknLwnCIgDRz2JQ7nf0w,7748
@@ -220,8 +221,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.30.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.30.2.dist-info/METADATA,sha256=eHDpKAYdqUR7OzLeXWFoYXh_gC2brf_cMK7hS02qHWI,7054
-port_ocean-0.30.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.30.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.30.2.dist-info/RECORD,,
+port_ocean-0.30.3.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.30.3.dist-info/METADATA,sha256=fgABnumshGTta5I9-Dm9mu1WHQa4uPmZrSx6E3VZ7do,7095
+port_ocean-0.30.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.30.3.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.30.3.dist-info/RECORD,,