port-ocean 0.28.14__py3-none-any.whl → 0.28.16__py3-none-any.whl

port_ocean/clients/port/mixins/entities.py

@@ -1,24 +1,24 @@
 import asyncio
+import json
 from typing import Any, Literal
 from urllib.parse import quote_plus
-import json
 
 import httpx
 from loguru import logger
-from port_ocean.context.ocean import ocean
+from starlette import status
+
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.types import RequestOptions, UserAgentType
 from port_ocean.clients.port.utils import (
-    handle_port_status_code,
     PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+    handle_port_status_code,
 )
+from port_ocean.context.ocean import ocean
 from port_ocean.core.models import (
     BulkUpsertResponse,
     Entity,
     PortAPIErrorMessage,
 )
-from starlette import status
-
 from port_ocean.helpers.metric.metric import MetricPhase, MetricType
 
 ENTITIES_BULK_SAMPLES_SIZE = 10
@@ -484,54 +484,89 @@ class EntityClientMixin:
         parameters_to_include: list[str] | None = None,
     ) -> list[Entity]:
         if query is None:
-            datasource_prefix = f"port-ocean/{self.auth.integration_type}/"
-            datasource_suffix = (
-                f"/{self.auth.integration_identifier}/{user_agent_type.value}"
-            )
-            logger.info(
-                f"Searching entities with datasource prefix: {datasource_prefix} and suffix: {datasource_suffix}"
-            )
+            return await self._search_entities_by_datasource_paginated(user_agent_type)
+
+        return await self._search_entities_by_query(
+            user_agent_type=user_agent_type,
+            query=query,
+            parameters_to_include=parameters_to_include,
+        )
+
+    async def _search_entities_by_datasource_paginated(
+        self, user_agent_type: UserAgentType
+    ) -> list[Entity]:
+        datasource_prefix = f"port-ocean/{self.auth.integration_type}/"
+        datasource_suffix = (
+            f"/{self.auth.integration_identifier}/{user_agent_type.value}"
+        )
+        logger.info(
+            f"Searching entities with datasource prefix: {datasource_prefix} and suffix: {datasource_suffix}"
+        )
+
+        next_from: str | None = None
+        aggregated_entities: list[Entity] = []
+        while True:
+            request_body: dict[str, Any] = {
+                "datasource_prefix": datasource_prefix,
+                "datasource_suffix": datasource_suffix,
+            }
+
+            if next_from:
+                request_body["from"] = next_from
 
             response = await self.client.post(
                 f"{self.auth.api_url}/blueprints/entities/datasource-entities",
-                json={
-                    "datasource_prefix": datasource_prefix,
-                    "datasource_suffix": datasource_suffix,
-                },
+                json=request_body,
                 headers=await self.auth.headers(user_agent_type),
                 extensions={"retryable": True},
             )
-        else:
-            default_query = {
-                "combinator": "and",
-                "rules": [
-                    {
-                        "property": "$datasource",
-                        "operator": "contains",
-                        "value": f"port-ocean/{self.auth.integration_type}/",
-                    },
-                    {
-                        "property": "$datasource",
-                        "operator": "contains",
-                        "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
-                    },
-                ],
-            }
+            handle_port_status_code(response)
+            response_json = response.json()
+            aggregated_entities.extend(
+                Entity.parse_obj(result) for result in response_json.get("entities", [])
+            )
+            next_from = response_json.get("next")
+            if not next_from:
+                break
 
-            if query.get("rules"):
-                query["rules"].extend(default_query["rules"])
+        return aggregated_entities
 
-            logger.info(f"Searching entities with custom query: {query}")
-            response = await self.client.post(
-                f"{self.auth.api_url}/entities/search",
-                json=query,
-                headers=await self.auth.headers(user_agent_type),
-                params={
-                    "exclude_calculated_properties": "true",
-                    "include": parameters_to_include or ["blueprint", "identifier"],
+    async def _search_entities_by_query(
+        self,
+        user_agent_type: UserAgentType,
+        query: dict[Any, Any],
+        parameters_to_include: list[str] | None,
+    ) -> list[Entity]:
+        default_query = {
+            "combinator": "and",
+            "rules": [
+                {
+                    "property": "$datasource",
+                    "operator": "contains",
+                    "value": f"port-ocean/{self.auth.integration_type}/",
                 },
-                extensions={"retryable": True},
-            )
+                {
+                    "property": "$datasource",
+                    "operator": "contains",
+                    "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
+                },
+            ],
+        }
+
+        if query.get("rules"):
+            query["rules"].extend(default_query["rules"])
+
+        logger.info(f"Searching entities with custom query: {query}")
+        response = await self.client.post(
+            f"{self.auth.api_url}/entities/search",
+            json=query,
+            headers=await self.auth.headers(user_agent_type),
+            params={
+                "exclude_calculated_properties": "true",
+                "include": parameters_to_include or ["blueprint", "identifier"],
+            },
+            extensions={"retryable": True},
+        )
 
         handle_port_status_code(response)
         return [Entity.parse_obj(result) for result in response.json()["entities"]]

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -808,5 +808,12 @@ class JQEntityProcessor(BaseEntityProcessor):
     ) -> dict[str, Any]:
         if isinstance(data, tuple):
             raw_data = json.loads(data[1])
-            return {items_to_parse_name: data[0], **raw_data}
+            return {
+                **(
+                    data[0]
+                    if items_to_parse_name in data[0]
+                    else {items_to_parse_name: data[0]}
+                ),
+                **raw_data,
+            }
         return data

port_ocean/core/integrations/mixins/sync_raw.py

@@ -123,7 +123,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
                 )
                 task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
-                tasks.append(resync_function_wrapper(task, resource_config.kind))
+                tasks.append(resync_function_wrapper(task, resource_config.kind, resource_config.port.items_to_parse))
 
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"

port_ocean/core/integrations/mixins/utils.py

@@ -56,7 +56,7 @@ def _process_path_type_items(
                         content = json.loads(f.read())
                     # Create a copy of the item with the content field
                     processed_item = item.copy()
-                    processed_item["content"] = content
+                    processed_item["file"]["content"] = content
                     processed_result.append(processed_item)
                 else:
                     # If file doesn't exist, keep the original item
@@ -97,12 +97,12 @@ def resync_error_handling() -> Generator[None, None, None]:
 
 
 async def resync_function_wrapper(
-    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
         validated_results = validate_result(results)
-        return _process_path_type_items(validated_results)
+        return _process_path_type_items(validated_results, items_to_parse)
 
 
 async def resync_generator_wrapper(
@@ -117,7 +117,7 @@ async def resync_generator_wrapper(
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
                         validated_result = validate_result(result)
-                        processed_result = _process_path_type_items(validated_result)
+                        processed_result = _process_path_type_items(validated_result,items_to_parse)
                         yield processed_result
                     else:
                         if items_to_parse:

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -179,7 +179,7 @@ async def test_search_entities_uses_datasource_route_when_query_is_none(
 ) -> None:
     """Test that search_entities uses datasource route when query is None"""
     mock_response = MagicMock()
-    mock_response.json.return_value = {"entities": []}
+    mock_response.json.return_value = {"entities": [], "next": None}
     mock_response.is_error = False
     mock_response.status_code = 200
     mock_response.headers = {}
@@ -208,8 +208,74 @@ async def test_search_entities_uses_datasource_route_when_query_is_none(
         == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
     )
 
-    expected_json = {
-        "datasource_prefix": "port-ocean/test-integration/",
-        "datasource_suffix": "/test-identifier/sync",
+    sent_json = call_args[1]["json"]
+    assert sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert sent_json["datasource_suffix"] == "/test-identifier/sync"
+
+
+async def test_search_entities_uses_datasource_route_when_query_is_none_two_pages(
+    entity_client: EntityClientMixin,
+) -> None:
+    """Test that search_entities uses datasource route when query is None"""
+    # First response with pagination token
+    mock_response_first = MagicMock()
+    mock_response_first.json.return_value = {
+        "entities": [
+            Entity(identifier="entity_1", blueprint="entity_1"),
+            Entity(identifier="entity_2", blueprint="entity_2"),
+        ],
+        "next": "next_page_token",
+    }
+    mock_response_first.is_error = False
+    mock_response_first.status_code = 200
+    mock_response_first.headers = {}
+
+    # Second response without pagination token (end of pagination)
+    mock_response_second = MagicMock()
+    mock_response_second.json.return_value = {
+        "entities": [Entity(identifier="entity_3", blueprint="entity_3")],
+        "next": None,
     }
-    assert call_args[1]["json"] == expected_json
+    mock_response_second.is_error = False
+    mock_response_second.status_code = 200
+    mock_response_second.headers = {}
+
+    # Mock the client to return different responses for each call
+    entity_client.client.post = AsyncMock(side_effect=[mock_response_first, mock_response_second])  # type: ignore
+    entity_client.auth.headers = AsyncMock(return_value={"Authorization": "Bearer test"})  # type: ignore
+
+    entity_client.auth.integration_type = "test-integration"
+    entity_client.auth.integration_identifier = "test-identifier"
+    entity_client.auth.api_url = "https://api.getport.io/v1"
+
+    mock_user_agent_type = MagicMock()
+    mock_user_agent_type.value = "sync"
+
+    entities = await entity_client.search_entities(
+        user_agent_type=mock_user_agent_type,
+        query=None,
+    )
+
+    # Should call the datasource-entities endpoint exactly twice for pagination
+    assert entity_client.client.post.await_count == 2
+    assert len(entities) == 3
+
+    # Check first call
+    first_call_args = entity_client.client.post.call_args_list[0]
+    assert (
+        first_call_args[0][0]
+        == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
+    )
+    first_sent_json = first_call_args[1]["json"]
+    assert first_sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert first_sent_json["datasource_suffix"] == "/test-identifier/sync"
+
+    # Check second call
+    second_call_args = entity_client.client.post.call_args_list[1]
+    assert (
+        second_call_args[0][0]
+        == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
+    )
+    second_sent_json = second_call_args[1]["json"]
+    assert second_sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert second_sent_json["datasource_suffix"] == "/test-identifier/sync"

{port_ocean-0.28.14.dist-info → port_ocean-0.28.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.28.14
+Version: 0.28.16
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.28.14.dist-info → port_ocean-0.28.16.dist-info}/RECORD

@@ -60,7 +60,7 @@ port_ocean/clients/port/authentication.py,sha256=ZO1Vw1nm-NlVUPPtPS5O4GGDvRmyS3v
 port_ocean/clients/port/client.py,sha256=hBXgU0CDseN2F-vn20JqowfVkcd6oSVmYrjn6t4TI6c,3616
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=aMCG4zePsMSMjMLiGrU37h5z5_ElfMzTcTvqvOI5wXY,4683
-port_ocean/clients/port/mixins/entities.py,sha256=X2NqH00eK6TMJ3a3QEQRVQlKHzyj5l1FiPkIhonnbPg,24234
+port_ocean/clients/port/mixins/entities.py,sha256=UiVssYYqJeHhrLahx1mW24B7oGVMZV2WVvUze_htuBk,25279
 port_ocean/clients/port/mixins/integrations.py,sha256=8EhGms1_iOaAPEexmHGF4PJaSSL4O09GtXr_Q8UyaJI,12049
 port_ocean/clients/port/mixins/migrations.py,sha256=vdL_A_NNUogvzujyaRLIoZEu5vmKDY2BxTjoGP94YzI,1467
 port_ocean/clients/port/mixins/organization.py,sha256=A2cP5V49KnjoAXxjmnm_XGth4ftPSU0qURNfnyUyS_Y,1041
@@ -101,7 +101,7 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=VEL_aEme7gGrS0jvKgYMcBPIX02ntW7lT5V1KG12pJA,32003
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=Z0njO1z2FUh8hX4GTdH7CmO0Afv-WeYRtxPs34mxKWE,32181
 port_ocean/core/handlers/entity_processor/jq_input_evaluator.py,sha256=hInrBMQBbOqZApB53CMLyTgLtC7FltRtqK9PqD0CgM0,4803
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
@@ -124,8 +124,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=Zga3fSxALuXmAMKmIS0hZYWRe22lSGhiSVFWUCI4f1U,40972
-port_ocean/core/integrations/mixins/utils.py,sha256=vfKBwJFKI2zN5VgimOZQzCXE7bbpSCkBP8QT-9hJiBs,14516
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=TrdgwaCNbE-VEJBDRqWmLQR_1Epz6y5nEIbPaLZbe3A,41009
+port_ocean/core/integrations/mixins/utils.py,sha256=hRIkvxw1bIfilq6n_2fqUiv_5OSKFMl6HmJfimH2hTA,14590
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -162,7 +162,7 @@ port_ocean/tests/cache/test_memory_cache.py,sha256=xlwIOBU0RVLYYJU83l_aoZDzZ6QID
 port_ocean/tests/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/clients/oauth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/clients/oauth/test_oauth_client.py,sha256=2XVMQUalDpiD539Z7_dk5BK_ngXQzsTmb2lNBsfEm9c,3266
-port_ocean/tests/clients/port/mixins/test_entities.py,sha256=_ZEQT7UMzg1gW8kH8oFjgRVcLwQb7dFac48Tw_vcCqk,8018
+port_ocean/tests/clients/port/mixins/test_entities.py,sha256=jOMJ3ICUlhjjPTo4q6qUrEjTKvXRLUE6KjqjdFiDRBY,10766
 port_ocean/tests/clients/port/mixins/test_integrations.py,sha256=vRt_EMsLozQC1LJNXxlvnHj3-FlOBGgAYxg5T0IAqtA,7621
 port_ocean/tests/clients/port/mixins/test_organization_mixin.py,sha256=zzKYz3h8dl4Z5A2QG_924m0y9U6XTth1XYOfwNrd_24,914
 port_ocean/tests/config/test_config.py,sha256=Rk4N-ldVSOfn1p23NzdVdfqUpPrqG2cMut4Sv-sAOrw,1023
@@ -211,8 +211,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.28.14.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.28.14.dist-info/METADATA,sha256=_w_kDhykkm9LosWg6OOe8HhARPQXRoL4duh__TvB68U,7016
-port_ocean-0.28.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.28.14.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.28.14.dist-info/RECORD,,
+port_ocean-0.28.16.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.28.16.dist-info/METADATA,sha256=F6ugYDYHUDQt0YMZ8cghhxj4skE6nM8c7hBrf5GCqZE,7016
+port_ocean-0.28.16.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.28.16.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.28.16.dist-info/RECORD,,