port-ocean 0.10.12__tar.gz → 0.12.0__tar.gz

{port_ocean-0.10.12 → port_ocean-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.10.12
+Version: 0.12.0
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -28,11 +28,11 @@ Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
 Requires-Dist: fastapi (>=0.100,<0.112)
 Requires-Dist: httpx (>=0.24.1,<0.28.0)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
+Requires-Dist: jq (>=1.8.0,<2.0.0)
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
 Requires-Dist: pydantic[dotenv] (>=1.10.8,<2.0.0)
 Requires-Dist: pydispatcher (>=2.0.7,<3.0.0)
 Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
-Requires-Dist: pyjq (>=2.6.0,<3.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pyyaml (>=6.0,<7.0)
 Requires-Dist: rich (>=13.4.1,<14.0.0) ; extra == "cli"

{port_ocean-0.10.12 → port_ocean-0.12.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,11 +1,10 @@
 import asyncio
-import functools
 from asyncio import Task
 from dataclasses import dataclass, field
 from functools import lru_cache
 from typing import Any, Optional
 
-import pyjq as jq  # type: ignore
+import jq  # type: ignore
 from loguru import logger
 
 from port_ocean.context.ocean import ocean
@@ -52,8 +51,8 @@ class JQEntityProcessor(BaseEntityProcessor):
         try:
             loop = asyncio.get_event_loop()
             compiled_pattern = self._compile(pattern)
-            first_value_callable = functools.partial(compiled_pattern.first, data)
-            return await loop.run_in_executor(None, first_value_callable)
+            func = compiled_pattern.input_value(data)
+            return await loop.run_in_executor(None, func.first)
         except Exception as exc:
             logger.debug(
                 f"Failed to search for pattern {pattern} in data {data}, {exc}"
@@ -62,10 +61,18 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
+        start_time = loop.time()
         compiled_pattern = self._compile(pattern)
-        first_value_callable = functools.partial(compiled_pattern.first, data)
-        value = await loop.run_in_executor(None, first_value_callable)
-
+        func = compiled_pattern.input_value(data)
+        compile_time = loop.time() - start_time
+        value = await loop.run_in_executor(None, func.first)
+        execute_time = loop.time() - start_time - compile_time
+        logger.debug(
+            f"Search for pattern {execute_time:.2f} seconds, compile time {compile_time:.2f} seconds",
+            pattern=pattern,
+            compile_time=compile_time,
+            execute_time=execute_time,
+        )
         if isinstance(value, bool):
             return value
 

{port_ocean-0.10.12 → port_ocean-0.12.0}/port_ocean/core/integrations/base.py

@@ -54,7 +54,7 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
         """
         Initializes handlers, establishes integration at the specified port, and starts the event listener.
         """
-        logger.info("Starting integration")
+        logger.info("Starting integration", integration_type=self.context.config.integration.type)
         if self.started:
             raise IntegrationAlreadyStartedException("Integration already started")
 

{port_ocean-0.10.12 → port_ocean-0.12.0}/port_ocean/log/sensetive.py

@@ -21,7 +21,7 @@ secret_patterns = {
     "GitHub": r"[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]",
     "Google Cloud Platform API Key": r"AIza[0-9A-Za-z\\-_]{35}",
     "Google Cloud Platform OAuth": r"[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com",
-    "Google (GCP) Service-account": r'"type": "service_account"',
+    "Google (GCP) Service-account": f'"type":{" "}"service_account"',
     "Google OAuth Access Token": r"ya29\\.[0-9A-Za-z\\-_]+",
     "Connection String": r"[a-zA-Z]+:\/\/[^/\s]+:[^/\s]+@[^/\s]+\/[^/\s]+",
 }

{port_ocean-0.10.12 → port_ocean-0.12.0}/port_ocean/ocean.py

@@ -96,7 +96,8 @@ class Ocean:
         loop = asyncio.get_event_loop()
         if interval is not None:
             logger.info(
-                f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)"
+                f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)",
+                scheduled_interval=interval,
             )
             repeated_function = repeat_every(
                 seconds=interval * 60,

port_ocean-0.12.0/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -0,0 +1,236 @@
+from typing import Any
+from unittest.mock import AsyncMock, Mock
+import pytest
+
+from port_ocean.context.ocean import PortOceanContext
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
+    JQEntityProcessor,
+)
+from port_ocean.core.ocean_types import CalculationResult
+from port_ocean.exceptions.core import EntityProcessorException
+
+
+@pytest.mark.asyncio
+class TestJQEntityProcessor:
+
+    @pytest.fixture
+    def mocked_processor(self, monkeypatch: Any) -> JQEntityProcessor:
+        mock_context = AsyncMock()
+        monkeypatch.setattr(PortOceanContext, "app", mock_context)
+        return JQEntityProcessor(mock_context)
+
+    async def test_compile(self, mocked_processor: JQEntityProcessor) -> None:
+        pattern = ".foo"
+        compiled = mocked_processor._compile(pattern)
+        assert compiled is not None
+
+    async def test_search(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        result = await mocked_processor._search(data, pattern)
+        assert result == "bar"
+
+    async def test_search_as_bool(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": True}
+        pattern = ".foo"
+        result = await mocked_processor._search_as_bool(data, pattern)
+        assert result is True
+
+    async def test_search_as_object(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": {"bar": "baz"}}
+        obj = {"foo": ".foo.bar"}
+        result = await mocked_processor._search_as_object(data, obj)
+        assert result == {"foo": "baz"}
+
+    async def test_get_mapped_entity(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        raw_entity_mappings = {"foo": ".foo"}
+        selector_query = '.foo == "bar"'
+        result = await mocked_processor._get_mapped_entity(
+            data, raw_entity_mappings, selector_query
+        )
+        assert result.entity == {"foo": "bar"}
+        assert result.did_entity_pass_selector is True
+
+    async def test_calculate_entity(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {"foo": "bar"}
+        raw_entity_mappings = {"foo": ".foo"}
+        selector_query = '.foo == "bar"'
+        result, errors = await mocked_processor._calculate_entity(
+            data, raw_entity_mappings, None, selector_query
+        )
+        assert len(result) == 1
+        assert result[0].entity == {"foo": "bar"}
+        assert result[0].did_entity_pass_selector is True
+        assert not errors
+
+    async def test_parse_items(self, mocked_processor: JQEntityProcessor) -> None:
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "identifier": ".foo",
+            "blueprint": ".foo",
+            "properties": {"foo": ".foo"},
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = '.foo == "bar"'
+        raw_results = [{"foo": "bar"}]
+        result = await mocked_processor._parse_items(mapping, raw_results)
+        assert isinstance(result, CalculationResult)
+        assert len(result.entity_selector_diff.passed) == 1
+        assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
+        assert not result.errors
+
+    async def test_in_operator(self, mocked_processor: JQEntityProcessor) -> None:
+        data = {
+            "key": "GetPort_SelfService",
+            "name": "GetPort SelfService",
+            "desc": "Test",
+            "qualifier": "VW",
+            "visibility": "public",
+            "selectionMode": "NONE",
+            "subViews": [
+                {
+                    "key": "GetPort_SelfService_Second",
+                    "name": "GetPort SelfService Second",
+                    "qualifier": "SVW",
+                    "selectionMode": "NONE",
+                    "subViews": [
+                        {
+                            "key": "GetPort_SelfService_Third",
+                            "name": "GetPort SelfService Third",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [],
+                            "referencedBy": [],
+                        },
+                        {
+                            "key": "Port_Test",
+                            "name": "Port Test",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [],
+                            "referencedBy": [],
+                        },
+                    ],
+                    "referencedBy": [],
+                },
+                {
+                    "key": "Python",
+                    "name": "Python",
+                    "qualifier": "SVW",
+                    "selectionMode": "NONE",
+                    "subViews": [
+                        {
+                            "key": "Time",
+                            "name": "Time",
+                            "qualifier": "SVW",
+                            "selectionMode": "NONE",
+                            "subViews": [
+                                {
+                                    "key": "port_*****",
+                                    "name": "port-*****",
+                                    "qualifier": "SVW",
+                                    "selectionMode": "NONE",
+                                    "subViews": [
+                                        {
+                                            "key": "port_*****:REferenced",
+                                            "name": "REferenced",
+                                            "qualifier": "VW",
+                                            "visibility": "public",
+                                            "originalKey": "REferenced",
+                                        }
+                                    ],
+                                    "referencedBy": [],
+                                }
+                            ],
+                            "referencedBy": [],
+                        }
+                    ],
+                    "referencedBy": [],
+                },
+                {
+                    "key": "GetPort_SelfService:Authentication_Application",
+                    "name": "Authentication Application",
+                    "desc": "For auth services",
+                    "qualifier": "APP",
+                    "visibility": "private",
+                    "selectedBranches": ["main"],
+                    "originalKey": "Authentication_Application",
+                },
+            ],
+            "referencedBy": [],
+        }
+        pattern = '.subViews | map(select((.qualifier | IN("VW", "SVW"))) | .key)'
+        result = await mocked_processor._search(data, pattern)
+        assert result == ["GetPort_SelfService_Second", "Python"]
+
+    async def test_failure_of_jq_expression(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo."
+        result = await mocked_processor._search(data, pattern)
+        assert result is None
+
+    async def test_search_as_object_failure(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": {"bar": "baz"}}
+        obj = {"foo": ".foo.bar."}
+        result = await mocked_processor._search_as_object(data, obj)
+        assert result == {"foo": None}
+
+    async def test_double_quotes_in_jq_expression(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = '"shalom"'
+        result = await mocked_processor._search(data, pattern)
+        assert result == "shalom"
+
+    async def test_search_as_bool_failure(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        with pytest.raises(
+            EntityProcessorException,
+            match="Expected boolean value, got <class 'str'> instead",
+        ):
+            await mocked_processor._search_as_bool(data, pattern)
+
+    @pytest.mark.timeout(3)
+    async def test_search_performance_10000(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        """
+        This test is to check the performance of the search method when called 10000 times.
+        """
+        data = {"foo": "bar"}
+        pattern = ".foo"
+        for _ in range(10000):
+            result = await mocked_processor._search(data, pattern)
+            assert result == "bar"
+
+    @pytest.mark.timeout(15)
+    async def test_parse_items_performance_10000(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        """
+        This test is to check the performance of the parse_items method when called 10000 times.
+        """
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "identifier": ".foo",
+            "blueprint": ".foo",
+            "properties": {"foo": ".foo"},
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = '.foo == "bar"'
+        raw_results = [{"foo": "bar"}]
+        for _ in range(10000):
+            result = await mocked_processor._parse_items(mapping, raw_results)
+            assert isinstance(result, CalculationResult)
+            assert len(result.entity_selector_diff.passed) == 1
+            assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
+            assert not result.errors

port_ocean-0.12.0/port_ocean/tests/helpers/fixtures.py

@@ -0,0 +1,46 @@
+from os import path
+from typing import Any, Callable, List, Tuple
+
+import pytest
+import pytest_asyncio
+
+from port_ocean.clients.port.client import PortClient
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.ocean import Ocean
+from port_ocean.tests.helpers.ocean_app import (
+    get_integation_resource_configs,
+    get_integration_ocean_app,
+)
+from port_ocean.tests.helpers.smoke_test import (
+    SmokeTestDetails,
+    get_port_client_for_fake_integration,
+    get_smoke_test_details,
+)
+
+
+@pytest.fixture
+def port_client_for_fake_integration() -> Tuple[SmokeTestDetails, PortClient]:
+    smoke_test_details = get_smoke_test_details()
+    port_client = get_port_client_for_fake_integration()
+
+    return smoke_test_details, port_client
+
+
+@pytest_asyncio.fixture
+def get_mocked_ocean_app(request: Any) -> Callable[[], Ocean]:
+    test_dir = path.join(path.dirname(request.module.__file__), "..")
+
+    def get_ocean_app() -> Ocean:
+        return get_integration_ocean_app(test_dir)
+
+    return get_ocean_app
+
+
+@pytest_asyncio.fixture
+def get_mock_ocean_resource_configs(request: Any) -> Callable[[], List[ResourceConfig]]:
+    module_dir = path.join(path.dirname(request.module.__file__), "..")
+
+    def get_ocean_resource_configs() -> List[ResourceConfig]:
+        return get_integation_resource_configs(module_dir)
+
+    return get_ocean_resource_configs

port_ocean-0.12.0/port_ocean/tests/helpers/integration.py

@@ -0,0 +1,31 @@
+from typing import List
+
+from loguru import logger
+
+from port_ocean.clients.port.client import PortClient
+
+
+async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None:
+    for blueprint in blueprints:
+        try:
+            bp = await client.get_blueprint(blueprint)
+            if bp is not None:
+                migration_id = await client.delete_blueprint(
+                    identifier=blueprint, delete_entities=True
+                )
+                if migration_id:
+                    await client.wait_for_migration_to_complete(
+                        migration_id=migration_id
+                    )
+        except Exception as bp_e:
+            logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}")
+    headers = await client.auth.headers()
+    try:
+        await client.client.delete(
+            f"{client.auth.api_url}/integrations/{client.integration_identifier}",
+            headers=headers,
+        )
+    except Exception as int_e:
+        logger.info(
+            f"Failed to delete integration ({client.integration_identifier}): {int_e}"
+        )

port_ocean-0.12.0/port_ocean/tests/helpers/port_client.py

@@ -0,0 +1,21 @@
+from typing import Union
+
+from port_ocean.clients.port.client import PortClient
+
+
+def get_port_client_for_integration(
+    client_id: str,
+    client_secret: str,
+    integration_identifier: str,
+    integration_type: str,
+    integration_version: str,
+    base_url: Union[str, None],
+) -> PortClient:
+    return PortClient(
+        base_url=base_url or "https://api.getport/io",
+        client_id=client_id,
+        client_secret=client_secret,
+        integration_identifier=integration_identifier,
+        integration_type=integration_type,
+        integration_version=integration_version,
+    )

port_ocean-0.12.0/port_ocean/tests/helpers/smoke_test.py

@@ -0,0 +1,82 @@
+from os import environ
+from port_ocean.clients.port.client import PortClient
+
+from loguru import logger
+from pydantic import BaseModel
+
+from port_ocean.tests.helpers.integration import cleanup_integration
+from port_ocean.tests.helpers.port_client import get_port_client_for_integration
+
+
+class SmokeTestDetails(BaseModel):
+    integration_identifier: str
+    blueprint_department: str
+    blueprint_person: str
+    integration_type: str
+    integration_version: str
+
+
+def get_smoke_test_details() -> SmokeTestDetails:
+    blueprint_department = "fake-department"
+    blueprint_person = "fake-person"
+    integration_identifier = "smoke-test-integration"
+    smoke_test_suffix = environ.get("SMOKE_TEST_SUFFIX")
+    if smoke_test_suffix is not None:
+        integration_identifier = f"{integration_identifier}-{smoke_test_suffix}"
+        blueprint_person = f"{blueprint_person}-{smoke_test_suffix}"
+        blueprint_department = f"{blueprint_department}-{smoke_test_suffix}"
+
+    return SmokeTestDetails(
+        integration_identifier=integration_identifier,
+        blueprint_person=blueprint_person,
+        blueprint_department=blueprint_department,
+        integration_version="0.1.4-dev",
+        integration_type="smoke-test",
+    )
+
+
+async def cleanup_smoke_test() -> None:
+    smoke_test_details = get_smoke_test_details()
+    client_id = environ.get("PORT_CLIENT_ID")
+    client_secret = environ.get("PORT_CLIENT_SECRET")
+
+    if not client_secret or not client_id:
+        assert False, "Missing port credentials"
+
+    base_url = environ.get("PORT_BASE_URL")
+    client = get_port_client_for_integration(
+        client_id,
+        client_secret,
+        smoke_test_details.integration_identifier,
+        smoke_test_details.integration_type,
+        smoke_test_details.integration_version,
+        base_url,
+    )
+
+    logger.info("Cleaning up fake integration")
+    await cleanup_integration(
+        client,
+        [smoke_test_details.blueprint_department, smoke_test_details.blueprint_person],
+    )
+    logger.info("Cleaning up fake integration complete")
+
+
+def get_port_client_for_fake_integration() -> PortClient:
+    smoke_test_details = get_smoke_test_details()
+    client_id = environ.get("PORT_CLIENT_ID")
+    client_secret = environ.get("PORT_CLIENT_SECRET")
+
+    if not client_secret or not client_id:
+        assert False, "Missing port credentials"
+
+    base_url = environ.get("PORT_BASE_URL")
+    client = get_port_client_for_integration(
+        client_id,
+        client_secret,
+        smoke_test_details.integration_identifier,
+        smoke_test_details.integration_type,
+        smoke_test_details.integration_version,
+        base_url,
+    )
+
+    return client

{port_ocean-0.10.12 → port_ocean-0.12.0}/port_ocean/tests/test_smoke.py

@@ -4,7 +4,10 @@ import pytest
 
 from port_ocean.clients.port.client import PortClient
 from port_ocean.clients.port.types import UserAgentType
-from port_ocean.tests.helpers.fixtures import SmokeTestDetails
+from port_ocean.tests.helpers.smoke_test import SmokeTestDetails
+
+
+pytestmark = pytest.mark.smoke
 
 
 @pytest.mark.skipif(

{port_ocean-0.10.12 → port_ocean-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.10.12"
+version = "0.12.0"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -36,7 +36,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = {version = "^1.10.8", extras = ["dotenv"]}
+pydantic = { version = "^1.10.8", extras = ["dotenv"] }
 loguru = "^0.7.0"
 pyyaml = "^6.0"
 werkzeug = ">=2.3.4,<4.0.0"
@@ -44,7 +44,7 @@ fastapi = ">=0.100,<0.112"
 uvicorn = ">=0.22,<0.31"
 confluent-kafka = "^2.1.1"
 httpx = ">=0.24.1,<0.28.0"
-pyjq = "^2.6.0"
+jq = "^1.8.0"
 urllib3 = ">=1.26.16,<3.0.0"
 six = "^1.16.0"
 pyhumps = "^3.8.0"
@@ -78,6 +78,7 @@ types-python-dateutil = "^2.9.0.20240316"
 types-pyyaml = "^6.0.12.10"
 types-toml = "^0.10.8.6"
 yamllint = "^1.32.0"
+pytest-timeout = "^2.3.1"
 
 [tool.towncrier]
 directory = "changelog"
@@ -87,35 +88,35 @@ package_dir = "."
 package = "port_ocean"
 underlines = [""]
 
-  [[tool.towncrier.type]]
-  directory = "breaking"
-  name = "Breaking Changes"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "breaking"
+name = "Breaking Changes"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "deprecation"
-  name = "Deprecations"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "deprecation"
+name = "Deprecations"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "feature"
-  name = "Features"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "feature"
+name = "Features"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "improvement"
-  name = "Improvements"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "improvement"
+name = "Improvements"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "bugfix"
-  name = "Bug Fixes"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "bugfix"
+name = "Bug Fixes"
+showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "doc"
-  name = "Improved Documentation"
-  showcontent = true
+[[tool.towncrier.type]]
+directory = "doc"
+name = "Improved Documentation"
+showcontent = true
 
 [tool.mypy]
 exclude = [
@@ -174,4 +175,7 @@ exclude = '''
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
-addopts = "-vv -n auto --ignore-glob='./integrations/*' ./port_ocean/tests"
+addopts = "-vv -n auto --durations=10 --color=yes --ignore-glob='./integrations/*' ./port_ocean/tests"
+markers = [
+    "smoke: Smoke tests (deselect with '-m \"not smoke\"')"
+]