port-ocean 0.30.2__tar.gz → 0.30.4__tar.gz

{port_ocean-0.30.2 → port_ocean-0.30.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.30.2
+Version: 0.30.4
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -33,6 +33,7 @@ Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: ijson (>=3.4.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.6)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
+Requires-Dist: jmespath (>=1.0.1,<2.0.0)
 Requires-Dist: jq (>=1.8.0,<2.0.0)
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)

{port_ocean-0.30.2 → port_ocean-0.30.4}/port_ocean/clients/port/mixins/entities.py

@@ -262,34 +262,36 @@ class EntityClientMixin:
         }
         error_entities = {error["index"]: error for error in result.get("errors", [])}
 
+        ocean.metrics.inc_metric(
+            name=MetricType.OBJECT_COUNT_NAME,
+            labels=[
+                ocean.metrics.current_resource_kind(),
+                MetricPhase.LOAD,
+                MetricPhase.LoadResult.LOADED,
+            ],
+            value=len(successful_entities),
+        )
+
+        ocean.metrics.inc_metric(
+            name=MetricType.OBJECT_COUNT_NAME,
+            labels=[
+                ocean.metrics.current_resource_kind(),
+                MetricPhase.LOAD,
+                MetricPhase.LoadResult.FAILED,
+            ],
+            value=len(error_entities),
+        )
+
         batch_results: list[tuple[bool | None, Entity]] = []
         for entity_index, original_entity in index_to_entity.items():
             reduced_entity = self._reduce_entity(original_entity)
             if entity_index in successful_entities:
-                ocean.metrics.inc_metric(
-                    name=MetricType.OBJECT_COUNT_NAME,
-                    labels=[
-                        ocean.metrics.current_resource_kind(),
-                        MetricPhase.LOAD,
-                        MetricPhase.LoadResult.LOADED,
-                    ],
-                    value=1,
-                )
                 success_entity = successful_entities[entity_index]
                 # Create a copy of the original entity with the new identifier
                 updated_entity = reduced_entity.copy()
                 updated_entity.identifier = success_entity["identifier"]
                 batch_results.append((True, updated_entity))
             elif entity_index in error_entities:
-                ocean.metrics.inc_metric(
-                    name=MetricType.OBJECT_COUNT_NAME,
-                    labels=[
-                        ocean.metrics.current_resource_kind(),
-                        MetricPhase.LOAD,
-                        MetricPhase.LoadResult.FAILED,
-                    ],
-                    value=1,
-                )
                 error = error_entities[entity_index]
                 if (
                     error.get("identifier") == "unknown"
@@ -357,16 +359,23 @@ class EntityClientMixin:
         blueprint = entities[0].blueprint
 
         identifier_counts = Counter((e.blueprint, e.identifier) for e in entities)
-        duplicate_count = sum(
-            count - 1 for count in identifier_counts.values() if count > 1
-        )
+        duplicates: dict[tuple[str, str], int] = {
+            key: cnt for key, cnt in identifier_counts.items() if cnt > 1
+        }
+
+        if duplicates:
+            # Total number of *extra* occurrences beyond the first appearance of each entity
+            duplicate_count = sum(cnt - 1 for cnt in duplicates.values())
+
+            # Show up to 5 example (blueprint, identifier) pairs to avoid noisy logs
+            duplicate_examples = list(duplicates)[:5]
 
-        if duplicate_count:
-            duplicate_examples = [
-                key for key, cnt in identifier_counts.items() if cnt > 1
-            ][:5]
             logger.warning(
-                f"Detected {duplicate_count} duplicate entities (by blueprint and identifier) that may not be ingested because an identical identifier existed. Examples: {duplicate_examples}"
+                "Detected duplicate entities (by blueprint and identifier) that may not be ingested because an identical identifier existed",
+                extra={
+                    "duplicate_examples": duplicate_examples,
+                    "duplicate_count": duplicate_count,
+                },
             )
 
         bulk_size = self.calculate_entities_batch_size(entities)

{port_ocean-0.30.2 → port_ocean-0.30.4}/port_ocean/helpers/async_client.py

@@ -5,6 +5,7 @@ from loguru import logger
 
 from port_ocean.helpers.retry import RetryTransport, RetryConfig
 from port_ocean.helpers.stream import Stream
+from typing import AsyncGenerator, List, Dict
 
 
 class OceanAsyncClient(httpx.AsyncClient):
@@ -56,3 +57,31 @@ class OceanAsyncClient(httpx.AsyncClient):
         response = await self.send(req, stream=True)
         response.raise_for_status()
         return Stream(response)
+
+
+class StreamingClientWrapper:
+    def __init__(self, http_client: OceanAsyncClient):
+        self._client = http_client
+
+    async def stream_json(
+        self,
+        url: str,
+        target_items_path: str,
+        **kwargs: Any,
+    ) -> AsyncGenerator[List[Dict[str, Any]], None]:
+        """
+        A wrapper that provides a unified async generator interface for both streaming
+        and non-streaming HTTP GET requests.
+
+        :param url: The URL to request.
+        :param target_items_path: A JMESPath string to extract the list of items
+                                  from the JSON response (e.g., 'results'). The wrapper
+                                  will automatically adapt this for the streaming parser.
+        :param kwargs: Additional arguments for the HTTP request.
+        """
+        # ijson needs a path to the items inside the array, e.g., "results.item"
+        streaming_path = f"{target_items_path}.item"
+        stream_response = await self._client.get_stream(url, **kwargs)
+        json_stream = stream_response.get_json_stream(target_items=streaming_path)
+        async for items_batch in json_stream:
+            yield items_batch

{port_ocean-0.30.2 → port_ocean-0.30.4}/port_ocean/helpers/stream.py

@@ -22,7 +22,10 @@ class Stream:
         if chunk_size is None:
             chunk_size = ocean_context.ocean.config.streaming.chunk_size
 
-        file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
+        streaming_location = ocean_context.ocean.config.streaming.location
+        os.makedirs(streaming_location, exist_ok=True)
+
+        file_name = f"{streaming_location}/{uuid.uuid4()}"
 
         crypt = Fernet(Fernet.generate_key())
 

port_ocean-0.30.4/port_ocean/tests/clients/test_streaming_wrapper.py

@@ -0,0 +1,85 @@
+import pytest
+from unittest.mock import AsyncMock
+from typing import Any, AsyncGenerator
+
+
+from port_ocean.helpers.async_client import OceanAsyncClient, StreamingClientWrapper
+from port_ocean.helpers.stream import Stream
+
+
+class MockStream:
+    """A mock of the port_ocean.helpers.stream.Stream class for testing."""
+
+    def __init__(self, data_chunks: list[list[dict[str, Any]]]) -> None:
+        self._data_chunks = data_chunks
+
+    async def get_json_stream(self, target_items: str) -> AsyncGenerator[Any, None]:
+        """An async generator that yields the data chunks."""
+        for chunk in self._data_chunks:
+            yield chunk
+
+
+@pytest.mark.asyncio
+async def test_stream_json_with_streaming_enabled() -> None:
+    """
+    Tests that when streaming is enabled, the wrapper uses the `get_stream` method
+    and yields data in batches.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    # Simulate receiving the data in two separate chunks/batches
+    mock_stream_instance = MockStream(
+        [[{"id": 1, "name": "one"}], [{"id": 2, "name": "two"}]]
+    )
+    mock_client.get_stream.return_value = mock_stream_instance
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    results = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    mock_client.get_stream.assert_called_once_with("http://test.com")
+    mock_client.get.assert_not_called()
+    assert results == [[{"id": 1, "name": "one"}], [{"id": 2, "name": "two"}]]
+
+
+@pytest.mark.asyncio
+async def test_stream_json_with_empty_results_streaming() -> None:
+    """
+    Tests that the wrapper handles an empty stream correctly in streaming mode.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    mock_stream_instance = MockStream([[]])  # Stream yields one empty batch
+    mock_client.get_stream.return_value = mock_stream_instance
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    results = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    assert results == [[]]
+
+
+@pytest.mark.asyncio
+async def test_stream_json_path_adaptation_for_streaming() -> None:
+    """
+    Tests that the wrapper correctly adapts the `target_items_path` for the
+    streaming parser (ijson) by appending `.item`.
+    """
+    # Arrange
+    mock_client = AsyncMock(spec=OceanAsyncClient)
+    mock_stream = AsyncMock(spec=Stream)
+    mock_client.get_stream.return_value = mock_stream
+
+    wrapper = StreamingClientWrapper(http_client=mock_client)
+
+    # Act
+    # We only need to trigger the call to check the arguments
+    _ = [item async for item in wrapper.stream_json("http://test.com", "results")]
+
+    # Assert
+    # Verify that get_json_stream was called with the modified path
+    mock_stream.get_json_stream.assert_called_once_with(target_items="results.item")

{port_ocean-0.30.2 → port_ocean-0.30.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.30.2"
+version = "0.30.4"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -69,6 +69,7 @@ pytest-cov = "^6.0.0"
 ijson = "^3.4.0"
 aiofiles = "^24.1.0"
 cryptography = "^44.0.1"
+jmespath = "^1.0.1"
 
 [tool.poetry.extras]
 cli = ["click", "rich", "cookiecutter", "jinja2-time"]