port-ocean 0.10.11__tar.gz → 0.11.0__tar.gz

{port_ocean-0.10.11 → port_ocean-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.10.11
+Version: 0.11.0
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -28,11 +28,11 @@ Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
 Requires-Dist: fastapi (>=0.100,<0.112)
 Requires-Dist: httpx (>=0.24.1,<0.28.0)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
+Requires-Dist: jq (>=1.8.0,<2.0.0)
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
 Requires-Dist: pydantic[dotenv] (>=1.10.8,<2.0.0)
 Requires-Dist: pydispatcher (>=2.0.7,<3.0.0)
 Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
-Requires-Dist: pyjq (>=2.6.0,<3.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pyyaml (>=6.0,<7.0)
 Requires-Dist: rich (>=13.4.1,<14.0.0) ; extra == "cli"

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,11 +1,10 @@
 import asyncio
-import functools
 from asyncio import Task
 from dataclasses import dataclass, field
 from functools import lru_cache
 from typing import Any, Optional
 
-import pyjq as jq  # type: ignore
+import jq  # type: ignore
 from loguru import logger
 
 from port_ocean.context.ocean import ocean
@@ -52,8 +51,8 @@ class JQEntityProcessor(BaseEntityProcessor):
         try:
             loop = asyncio.get_event_loop()
             compiled_pattern = self._compile(pattern)
-            first_value_callable = functools.partial(compiled_pattern.first, data)
-            return await loop.run_in_executor(None, first_value_callable)
+            func = compiled_pattern.input_value(data)
+            return await loop.run_in_executor(None, func.first)
         except Exception as exc:
             logger.debug(
                 f"Failed to search for pattern {pattern} in data {data}, {exc}"
@@ -62,10 +61,18 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
+        start_time = loop.time()
         compiled_pattern = self._compile(pattern)
-        first_value_callable = functools.partial(compiled_pattern.first, data)
-        value = await loop.run_in_executor(None, first_value_callable)
-
+        func = compiled_pattern.input_value(data)
+        compile_time = loop.time() - start_time
+        value = await loop.run_in_executor(None, func.first)
+        execute_time = loop.time() - start_time - compile_time
+        logger.debug(
+            f"Search for pattern {execute_time:.2f} seconds, compile time {compile_time:.2f} seconds",
+            pattern=pattern,
+            compile_time=compile_time,
+            execute_time=execute_time,
+        )
         if isinstance(value, bool):
             return value
 

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/core/integrations/base.py

@@ -54,7 +54,7 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
         """
         Initializes handlers, establishes integration at the specified port, and starts the event listener.
         """
-        logger.info("Starting integration")
+        logger.info("Starting integration", integration_type=self.context.config.integration.type)
         if self.started:
             raise IntegrationAlreadyStartedException("Integration already started")
 

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -456,7 +456,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
                         creation_results.append(await task)
             except asyncio.CancelledError as e:
-                logger.warning("Resync aborted successfully")
+                logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
+                raise
             else:
                 if not did_fetched_current_state:
                     logger.warning(
@@ -489,3 +490,4 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         {"before": entities_at_port, "after": flat_created_entities},
                         user_agent_type,
                     )
+                    logger.info("Resync finished successfully")

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/ocean.py

@@ -82,6 +82,10 @@ class Ocean:
             try:
                 await self.integration.sync_raw_all()
                 await self.resync_state_updater.update_after_resync()
+            except asyncio.CancelledError:
+                logger.warning(
+                    "resync was cancelled by the scheduled resync, skipping state update"
+                )
             except Exception as e:
                 await self.resync_state_updater.update_after_resync(
                     IntegrationStateStatus.Failed
@@ -92,7 +96,8 @@ class Ocean:
         loop = asyncio.get_event_loop()
         if interval is not None:
             logger.info(
-                f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)"
+                f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)",
+                scheduled_interval=interval,
             )
             repeated_function = repeat_every(
                 seconds=interval * 60,

port_ocean-0.11.0/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -0,0 +1,236 @@
+from typing import Any
+from unittest.mock import AsyncMock, Mock
+import pytest
+
+from port_ocean.context.ocean import PortOceanContext
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
+    JQEntityProcessor,
+)
+from port_ocean.core.ocean_types import CalculationResult
+from port_ocean.exceptions.core import EntityProcessorException
+
+
+@pytest.mark.asyncio
+class TestJQEntityProcessor:
+
+    @pytest.fixture
+    def mocked_processor(self, monkeypatch: Any) -> JQEntityProcessor:
+        mock_context = AsyncMock()
+        monkeypatch.setattr(PortOceanContext, "app", mock_context)
+        return JQEntityProcessor(mock_context)
+
+    async def test_compile(self, mocked_processor: JQEntityProcessor) -> None:
+        pattern = ".foo"
+        compiled = mocked_processor._compile(pattern)
+        assert compiled is not None
+
+    async def test_search(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        result = await mocked_processor._search(data, pattern)
+        assert result == "bar"
+
+    async def test_search_as_bool(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": True}
+        pattern = ".foo"
+        result = await mocked_processor._search_as_bool(data, pattern)
+        assert result is True
+
+    async def test_search_as_object(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": {"bar": "baz"}}
+        obj = {"foo": ".foo.bar"}
+        result = await mocked_processor._search_as_object(data, obj)
+        assert result == {"foo": "baz"}
+
+    async def test_get_mapped_entity(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        raw_entity_mappings = {"foo": ".foo"}
+        selector_query = '.foo == "bar"'
+        result = await mocked_processor._get_mapped_entity(
+            data, raw_entity_mappings, selector_query
+        )
+        assert result.entity == {"foo": "bar"}
+        assert result.did_entity_pass_selector is True
+
+    async def test_calculate_entity(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        raw_entity_mappings = {"foo": ".foo"}
+        selector_query = '.foo == "bar"'
+        result, errors = await mocked_processor._calculate_entity(
+            data, raw_entity_mappings, None, selector_query
+        )
+        assert len(result) == 1
+        assert result[0].entity == {"foo": "bar"}
+        assert result[0].did_entity_pass_selector is True
+        assert not errors
+
+    async def test_parse_items(self, mocked_processor: JQEntityProcessor) -> None:
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "identifier": ".foo",
+            "blueprint": ".foo",
+            "properties": {"foo": ".foo"},
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = '.foo == "bar"'
+        raw_results = [{"foo": "bar"}]
+        result = await mocked_processor._parse_items(mapping, raw_results)
+        assert isinstance(result, CalculationResult)
+        assert len(result.entity_selector_diff.passed) == 1
+        assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
+        assert not result.errors
+
+    async def test_in_operator(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {
+            "key": "GetPort_SelfService",
+            "name": "GetPort SelfService",
+            "desc": "Test",
+            "qualifier": "VW",
+            "visibility": "public",
+            "selectionMode": "NONE",
+            "subViews": [
+                {
+                    "key": "GetPort_SelfService_Second",
+                    "name": "GetPort SelfService Second",
+                    "qualifier": "SVW",
+                    "selectionMode": "NONE",
+                    "subViews": [
+                        {
+                            "key": "GetPort_SelfService_Third",
+                            "name": "GetPort SelfService Third",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [],
+                            "referencedBy": [],
+                        },
+                        {
+                            "key": "Port_Test",
+                            "name": "Port Test",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [],
+                            "referencedBy": [],
+                        },
+                    ],
+                    "referencedBy": [],
+                },
+                {
+                    "key": "Python",
+                    "name": "Python",
+                    "qualifier": "SVW",
+                    "selectionMode": "NONE",
+                    "subViews": [
+                        {
+                            "key": "Time",
+                            "name": "Time",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [
+                                {
+                                    "key": "port_*****",
+                                    "name": "port-*****",
+                                    "qualifier": "SVW",
+                                    "selectionMode": "NONE",
+                                    "subViews": [
+                                        {
+                                            "key": "port_*****:REferenced",
+                                            "name": "REferenced",
+                                            "qualifier": "VW",
+                                            "visibility": "public",
+                                            "originalKey": "REferenced",
+                                        }
+                                    ],
+                                    "referencedBy": [],
+                                }
+                            ],
+                            "referencedBy": [],
+                        }
+                    ],
+                    "referencedBy": [],
+                },
+                {
+                    "key": "GetPort_SelfService:Authentication_Application",
+                    "name": "Authentication Application",
+                    "desc": "For auth services",
+                    "qualifier": "APP",
+                    "visibility": "private",
+                    "selectedBranches": ["main"],
+                    "originalKey": "Authentication_Application",
+                },
+            ],
+            "referencedBy": [],
+        }
+        pattern = '.subViews | map(select((.qualifier | IN("VW", "SVW"))) | .key)'
+        result = await mocked_processor._search(data, pattern)
+        assert result == ["GetPort_SelfService_Second", "Python"]
+
+    async def test_failure_of_jq_expression(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo."
+        result = await mocked_processor._search(data, pattern)
+        assert result is None
+
+    async def test_search_as_object_failure(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": {"bar": "baz"}}
+        obj = {"foo": ".foo.bar."}
+        result = await mocked_processor._search_as_object(data, obj)
+        assert result == {"foo": None}
+
+    async def test_double_quotes_in_jq_expression(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = '"shalom"'
+        result = await mocked_processor._search(data, pattern)
+        assert result == "shalom"
+
+    async def test_search_as_bool_failure(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        with pytest.raises(
+            EntityProcessorException,
+            match="Expected boolean value, got <class 'str'> instead",
+        ):
+            await mocked_processor._search_as_bool(data, pattern)
+
+    @pytest.mark.timeout(3)
+    async def test_search_performance_10000(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        """
+        This test is to check the performance of the search method when called 10000 times.
+        """
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        for _ in range(10000):
+            result = await mocked_processor._search(data, pattern)
+            assert result == "bar"
+
+    @pytest.mark.timeout(15)
+    async def test_parse_items_performance_10000(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        """
+        This test is to check the performance of the parse_items method when called 10000 times.
+        """
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "identifier": ".foo",
+            "blueprint": ".foo",
+            "properties": {"foo": ".foo"},
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = '.foo == "bar"'
+        raw_results = [{"foo": "bar"}]
+        for _ in range(10000):
+            result = await mocked_processor._parse_items(mapping, raw_results)
+            assert isinstance(result, CalculationResult)
+            assert len(result.entity_selector_diff.passed) == 1
+            assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
+            assert not result.errors

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/tests/helpers/fixtures.py

@@ -2,6 +2,7 @@ from os import environ, path
 from typing import Any, AsyncGenerator, Callable, List, Tuple, Union
 
 import pytest_asyncio
+from loguru import logger
 from pydantic import BaseModel
 
 from port_ocean.clients.port.client import PortClient
@@ -33,15 +34,28 @@ def get_port_client_for_integration(
 
 async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None:
     for blueprint in blueprints:
-        bp = await client.get_blueprint(blueprint)
-        if bp is not None:
-            migration_id = await client.delete_blueprint(
-                identifier=blueprint, delete_entities=True
-            )
-            if migration_id:
-                await client.wait_for_migration_to_complete(migration_id=migration_id)
+        try:
+            bp = await client.get_blueprint(blueprint)
+            if bp is not None:
+                migration_id = await client.delete_blueprint(
+                    identifier=blueprint, delete_entities=True
+                )
+                if migration_id:
+                    await client.wait_for_migration_to_complete(
+                        migration_id=migration_id
+                    )
+        except Exception as bp_e:
+            logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}")
     headers = await client.auth.headers()
-    await client.client.delete(f"{client.auth.api_url}/integrations", headers=headers)
+    try:
+        await client.client.delete(
+            f"{client.auth.api_url}/integrations/{client.integration_identifier}",
+            headers=headers,
+        )
+    except Exception as int_e:
+        logger.info(
+            f"Failed to delete integration ({client.integration_identifier}): {int_e}"
+        )
 
 
 class SmokeTestDetails(BaseModel):

{port_ocean-0.10.11 → port_ocean-0.11.0}/port_ocean/tests/test_smoke.py

@@ -7,6 +7,9 @@ from port_ocean.clients.port.types import UserAgentType
 from port_ocean.tests.helpers.fixtures import SmokeTestDetails
 
 
+pytestmark = pytest.mark.smoke
+
+
 @pytest.mark.skipif(
     environ.get("SMOKE_TEST_SUFFIX", None) is None,
     reason="You need to run the fake integration once",

{port_ocean-0.10.11 → port_ocean-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.10.11"
+version = "0.11.0"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -36,7 +36,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = {version = "^1.10.8", extras = ["dotenv"]}
+pydantic = { version = "^1.10.8", extras = ["dotenv"] }
 loguru = "^0.7.0"
 pyyaml = "^6.0"
 werkzeug = ">=2.3.4,<4.0.0"
@@ -44,7 +44,7 @@ fastapi = ">=0.100,<0.112"
 uvicorn = ">=0.22,<0.31"
 confluent-kafka = "^2.1.1"
 httpx = ">=0.24.1,<0.28.0"
-pyjq = "^2.6.0"
+jq = "^1.8.0"
 urllib3 = ">=1.26.16,<3.0.0"
 six = "^1.16.0"
 pyhumps = "^3.8.0"
@@ -78,6 +78,7 @@ types-python-dateutil = "^2.9.0.20240316"
 types-pyyaml = "^6.0.12.10"
 types-toml = "^0.10.8.6"
 yamllint = "^1.32.0"
+pytest-timeout = "^2.3.1"
 
 [tool.towncrier]
 directory = "changelog"
@@ -87,35 +88,35 @@ package_dir = "."
 package = "port_ocean"
 underlines = [""]
 
-  [[tool.towncrier.type]]
-  directory = "breaking"
-  name = "Breaking Changes"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "breaking"
+name = "Breaking Changes"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "deprecation"
-  name = "Deprecations"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "deprecation"
+name = "Deprecations"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "feature"
-  name = "Features"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "feature"
+name = "Features"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "improvement"
-  name = "Improvements"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "improvement"
+name = "Improvements"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "bugfix"
-  name = "Bug Fixes"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "bugfix"
+name = "Bug Fixes"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "doc"
-  name = "Improved Documentation"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "doc"
+name = "Improved Documentation"
+showcontent = true
 
 [tool.mypy]
 exclude = [
@@ -174,4 +175,4 @@ exclude = '''
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
-addopts = "-vv -n auto --ignore-glob='./integrations/*' ./port_ocean/tests"
+addopts = "-vv -n auto --durations=10 --color=yes --ignore-glob='./integrations/*' ./port_ocean/tests"

port_ocean-0.10.11/port_ocean/tests/test_sample.py

@@ -1,2 +0,0 @@
-def test_example() -> None:
-    assert 1 == 1