port-ocean 0.30.0__tar.gz → 0.30.2__tar.gz

{port_ocean-0.30.0 → port_ocean-0.30.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.30.0
+Version: 0.30.2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.30.0 → port_ocean-0.30.2}/port_ocean/clients/port/mixins/entities.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from collections import Counter
 from typing import Any, Literal
 from urllib.parse import quote_plus
 
@@ -355,6 +356,19 @@ class EntityClientMixin:
         entities_results: list[tuple[bool, Entity]] = []
         blueprint = entities[0].blueprint
 
+        identifier_counts = Counter((e.blueprint, e.identifier) for e in entities)
+        duplicate_count = sum(
+            count - 1 for count in identifier_counts.values() if count > 1
+        )
+
+        if duplicate_count:
+            duplicate_examples = [
+                key for key, cnt in identifier_counts.items() if cnt > 1
+            ][:5]
+            logger.warning(
+                f"Detected {duplicate_count} duplicate entities (by blueprint and identifier) that may not be ingested because an identical identifier existed. Examples: {duplicate_examples}"
+            )
+
         bulk_size = self.calculate_entities_batch_size(entities)
         bulks = [
             entities[i : i + bulk_size] for i in range(0, len(entities), bulk_size)

{port_ocean-0.30.0 → port_ocean-0.30.2}/port_ocean/core/defaults/initialize.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import Type, Any
+from typing import Any, Type
 
 import httpx
 from loguru import logger
@@ -114,12 +114,67 @@ async def _initialize_required_integration_settings(
         )
 
 
+async def _mapped_blueprints_exist(
+    port_client: PortClient,
+    defaults: Defaults | None = None,
+    has_provision_feature_flag: bool = False,
+) -> bool:
+    if not defaults:
+        return True
+
+    integration = await port_client.get_current_integration(
+        should_log=False,
+        should_raise=False,
+        has_provision_feature_flag=has_provision_feature_flag,
+    )
+    integration_config = integration.get("config", {})
+    resources = integration_config.get("resources", [])
+
+    if not isinstance(resources, list):
+        return True
+
+    mapped_blueprints = []
+    for resource in resources:
+        blueprint = (
+            resource.get("port", {})
+            .get("entity", {})
+            .get("mappings", {})
+            .get("blueprint")
+        )
+        if blueprint:
+            if (
+                isinstance(blueprint, str)
+                and blueprint.startswith('"')
+                and blueprint.endswith('"')
+            ):
+                blueprint = blueprint.strip('"')
+            mapped_blueprints.append({"identifier": blueprint})
+
+    if not mapped_blueprints:
+        return True
+
+    existing_blueprints, _ = await gather_and_split_errors_from_results(
+        [
+            port_client.get_blueprint(blueprint["identifier"], should_log=False)
+            for blueprint in mapped_blueprints
+        ],
+        lambda item: isinstance(item, Blueprint),
+    )
+
+    if len(existing_blueprints) != len(mapped_blueprints):
+        return False
+
+    return True
+
+
 async def _create_resources(
     port_client: PortClient,
     defaults: Defaults | None = None,
+    has_provision_feature_flag: bool = False,
 ) -> None:
     if not defaults:
         return
+
     creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps(
         defaults.blueprints
     )
@@ -132,7 +187,13 @@ async def _create_resources(
         lambda item: isinstance(item, Blueprint),
     )
 
-    if blueprints_results:
+    mapped_blueprints_exist = await _mapped_blueprints_exist(
+        port_client,
+        defaults,
+        has_provision_feature_flag,
+    )
+
+    if blueprints_results or mapped_blueprints_exist:
         logger.info(
             f"Blueprints already exist: {[result.identifier for result in blueprints_results]}. Skipping integration default creation..."
         )
@@ -184,15 +245,16 @@ async def _create_resources(
             )
         )
 
-        created_scorecards, scorecards_errors = (
-            await gather_and_split_errors_from_results(
-                (
-                    port_client.create_scorecard(
-                        blueprint_scorecards["blueprint"], action, should_log=False
-                    )
-                    for blueprint_scorecards in defaults.scorecards
-                    for action in blueprint_scorecards["data"]
+        (
+            created_scorecards,
+            scorecards_errors,
+        ) = await gather_and_split_errors_from_results(
+            (
+                port_client.create_scorecard(
+                    blueprint_scorecards["blueprint"], action, should_log=False
                 )
+                for blueprint_scorecards in defaults.scorecards
+                for action in blueprint_scorecards["data"]
             )
         )
 
@@ -286,7 +348,7 @@ async def _initialize_defaults(
         return
     try:
         logger.info("Found default resources, starting creation process")
-        await _create_resources(port_client, defaults)
+        await _create_resources(port_client, defaults, has_provision_feature_flag)
     except AbortDefaultCreationError as e:
         logger.warning(
             f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"

{port_ocean-0.30.0 → port_ocean-0.30.2}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,19 +1,27 @@
 import asyncio
+import json
+import re
 from asyncio import Task
 from dataclasses import dataclass, field
 from functools import lru_cache
-import json
 from typing import Any, Optional
+
 import jq  # type: ignore
 from loguru import logger
+
 from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
+from port_ocean.core.handlers.entity_processor.jq_input_evaluator import (
+    InputClassifyingResult,
+    can_expression_run_with_no_input,
+    classify_input,
+)
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RAW_ITEM,
-    EntitySelectorDiff,
     CalculationResult,
+    EntitySelectorDiff,
 )
 from port_ocean.core.utils.utils import (
     gather_and_split_errors_from_results,
@@ -21,11 +29,6 @@ from port_ocean.core.utils.utils import (
 )
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
-from port_ocean.core.handlers.entity_processor.jq_input_evaluator import (
-    InputClassifyingResult,
-    classify_input,
-    can_expression_run_with_no_input,
-)
 
 
 class ExampleStates:
@@ -92,8 +95,29 @@ class JQEntityProcessor(BaseEntityProcessor):
     searching for data in dictionaries, and transforming data based on object mappings.
     """
 
+    @staticmethod
+    def _format_filter(filter: str) -> str:
+        """
+        Convert single quotes to double quotes in JQ expressions.
+        Only replaces single quotes that are opening or closing string delimiters,
+        not single quotes that are part of string content.
+        """
+        # Escape single quotes only if they are opening or closing a string
+        # Pattern matches:
+        # - Single quote at start of string or after whitespace (opening quote)
+        # - Single quote before whitespace or end of string (closing quote)
+        # Uses negative lookahead/lookbehind to avoid replacing quotes inside strings
+        # \1 and \2 will be empty for the alternative that didn't match, so \1"\2 works for both cases
+        # This matches the TypeScript pattern: /(^|\s)'(?!\s|")|(?<!\s|")'(\s|$)/g
+        formatted_filter = re.sub(
+            r'(^|\s)\'(?!\s|")|(?<!\s|")\'(\s|$)', r'\1"\2', filter
+        )
+        return formatted_filter
+
     @lru_cache
     def _compile(self, pattern: str) -> Any:
+        # Convert single quotes to double quotes for JQ compatibility
+        pattern = self._format_filter(pattern)
         if not ocean.config.allow_environment_variables_jq_access:
             pattern = "def env: {}; {} as $ENV | " + pattern
         return jq.compile(pattern)
@@ -119,7 +143,6 @@ class JQEntityProcessor(BaseEntityProcessor):
         missing_required_fields: bool,
         entity_mapping_fault_counter: int,
     ) -> None:
-
         if len(entity_misconfigurations) > 0:
             logger.info(
                 f"Unable to find valid data for: {entity_misconfigurations} (null, missing, or misconfigured)"
@@ -444,7 +467,6 @@ class JQEntityProcessor(BaseEntityProcessor):
         key: str,
         value: dict[str, Any],
     ) -> None:
-
         if key in ["properties", "relations"]:
             mapping_dicts: dict[InputClassifyingResult, dict[str, Any]] = {
                 InputClassifyingResult.SINGLE: {},

port_ocean-0.30.2/port_ocean/tests/core/defaults/test_initialize.py

@@ -0,0 +1,219 @@
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from port_ocean.clients.port.client import PortClient
+from port_ocean.clients.port.types import UserAgentType
+from port_ocean.core.defaults.common import Defaults
+from port_ocean.core.defaults.initialize import _create_resources
+from port_ocean.core.models import Blueprint
+
+
+@pytest.fixture
+def mock_defaults() -> Defaults:
+    """Create a mock Defaults object with sample data."""
+    return Defaults(
+        blueprints=[
+            {
+                "identifier": "blueprint1",
+                "title": "Blueprint 1",
+                "icon": "Microservice",
+                "schema": {"type": "object", "properties": {}},
+                "relations": {},
+                "calculationProperties": {},
+                "mirrorProperties": {},
+                "aggregationProperties": {},
+                "teamInheritance": {},
+                "ownership": {},
+            },
+            {
+                "identifier": "blueprint2",
+                "title": "Blueprint 2",
+                "icon": "Microservice",
+                "schema": {"type": "object", "properties": {}},
+                "relations": {},
+                "calculationProperties": {},
+                "mirrorProperties": {},
+                "aggregationProperties": {},
+                "teamInheritance": {},
+                "ownership": {},
+            },
+        ],
+        actions=[
+            {"identifier": "action1", "title": "Action 1"},
+            {"identifier": "action2", "title": "Action 2"},
+        ],
+        scorecards=[
+            {"blueprint": "blueprint1", "data": [{"identifier": "scorecard1"}]},
+        ],
+        pages=[
+            {"identifier": "page1", "title": "Page 1"},
+        ],
+    )
+
+
+@pytest.fixture
+def mock_port_client() -> PortClient:
+    """Create a mock PortClient."""
+    mock_client = MagicMock(spec=PortClient)
+    mock_client.get_blueprint = AsyncMock()
+    mock_client.get_current_integration = AsyncMock()
+    mock_client.create_blueprint = AsyncMock()
+    mock_client.patch_blueprint = AsyncMock()
+    mock_client.create_action = AsyncMock()
+    mock_client.create_scorecard = AsyncMock()
+    mock_client.create_page = AsyncMock()
+    return mock_client
+
+
+@pytest.mark.asyncio
+async def test_create_resources_all_mapped_blueprints_exist(
+    mock_port_client: PortClient, mock_defaults: Defaults
+) -> None:
+    """
+    Test case 1: All mapped blueprints exist in Port and they are different from default ones.
+    Should skip creation of all resources.
+
+    Flow:
+    1. Get_blueprint is called for creation_stage default blueprints
+       - These should NOT exist (raise exception) so blueprints_results = []
+    2. _mapped_blueprints_exist calls get_blueprint for mapped blueprints
+       - These SHOULD exist (return Blueprint) so mapped_blueprints_exist = True
+    3. Since mapped_blueprints_exist is True, we skip creation
+    """
+    mock_port_client.get_current_integration.return_value = {  # type: ignore[attr-defined]
+        "config": {
+            "resources": [
+                {
+                    "port": {
+                        "entity": {
+                            "mappings": {
+                                "blueprint": "blueprint1test",
+                            }
+                        }
+                    }
+                },
+                {
+                    "port": {
+                        "entity": {
+                            "mappings": {
+                                "blueprint": "blueprint2test",
+                            }
+                        }
+                    }
+                },
+            ]
+        }
+    }
+
+    # Track call order: first 2 calls are for creation_stage, next 2 are for mapped blueprints
+    call_count = 0
+
+    async def get_blueprint_side_effect(
+        identifier: str, should_log: bool = True
+    ) -> Blueprint:
+        nonlocal call_count
+        call_count += 1
+
+        # First 2 calls are for creation_stage blueprints
+        # These should NOT exist to make blueprints_results = []
+        if call_count <= 2:
+            raise Exception(f"Blueprint {identifier} not found")
+
+        # Next 2 calls are for mapped blueprints
+        # These SHOULD exist to make mapped_blueprints_exist = True
+        return Blueprint.parse_obj(
+            {
+                "identifier": identifier,
+                "title": f"Blueprint {identifier}",
+                "team": None,
+                "schema": {"type": "object", "properties": {}},
+                "relations": {},
+            }
+        )
+
+    mock_port_client.get_blueprint.side_effect = get_blueprint_side_effect  # type: ignore[attr-defined]
+
+    # Execute
+    await _create_resources(
+        mock_port_client, mock_defaults, has_provision_feature_flag=False
+    )
+
+    # Assert: Should not create any blueprints, actions, scorecards, or pages
+    mock_port_client.create_blueprint.assert_not_called()  # type: ignore[attr-defined]
+    mock_port_client.patch_blueprint.assert_not_called()  # type: ignore[attr-defined]
+    mock_port_client.create_action.assert_not_called()  # type: ignore[attr-defined]
+    mock_port_client.create_scorecard.assert_not_called()  # type: ignore[attr-defined]
+    mock_port_client.create_page.assert_not_called()  # type: ignore[attr-defined]
+    assert mock_port_client.get_blueprint.call_count == 4  # type: ignore[attr-defined]
+
+
+@pytest.mark.asyncio
+async def test_create_resources_no_mapped_blueprints_exist(
+    mock_port_client: PortClient, mock_defaults: Defaults
+) -> None:
+    """
+    Test case 2: No mapped blueprints exist in Port (they are in config but don't exist).
+    Should create all blueprints, actions, scorecards, and pages from defaults.
+
+    Flow:
+    1. Get_blueprint is called for creation_stage default blueprints
+       - These should NOT exist (raise exception) so blueprints_results = []
+    2. _mapped_blueprints_exist calls get_blueprint for mapped blueprints
+       - These should NOT exist (raise exception) so mapped_blueprints_exist = False
+    3. Since both are False/empty, we create all resources
+    """
+    mock_port_client.get_current_integration.return_value = {  # type: ignore[attr-defined]
+        "config": {
+            "resources": [
+                {
+                    "port": {
+                        "entity": {
+                            "mappings": {
+                                "blueprint": "blueprint1",
+                            }
+                        }
+                    }
+                },
+                {
+                    "port": {
+                        "entity": {
+                            "mappings": {
+                                "blueprint": "blueprint2",
+                            }
+                        }
+                    }
+                },
+            ]
+        }
+    }
+
+    # Mock get_blueprint to always raise exception (blueprints don't exist)
+    # This simulates that no blueprints exist in Port
+    async def get_blueprint_side_effect(
+        identifier: str, should_log: bool = True
+    ) -> Blueprint:
+        raise Exception(f"Blueprint {identifier} not found")
+
+    mock_port_client.get_blueprint.side_effect = get_blueprint_side_effect  # type: ignore[attr-defined]
+
+    # Mock create_blueprint to return created blueprint
+    async def create_blueprint_side_effect(
+        blueprint: dict[str, Any], user_agent_type: UserAgentType | None = None
+    ) -> dict[str, Any]:
+        return {"identifier": blueprint["identifier"], **blueprint}
+
+    mock_port_client.create_blueprint.side_effect = create_blueprint_side_effect  # type: ignore[attr-defined]
+
+    # Execute
+    await _create_resources(
+        mock_port_client, mock_defaults, has_provision_feature_flag=False
+    )
+
+    # Assert
+    assert mock_port_client.create_blueprint.call_count == 2  # type: ignore[attr-defined]
+    assert mock_port_client.patch_blueprint.call_count >= 2  # type: ignore[attr-defined]
+    assert mock_port_client.create_action.call_count == 2  # type: ignore[attr-defined]
+    assert mock_port_client.create_scorecard.call_count == 1  # type: ignore[attr-defined]
+    assert mock_port_client.create_page.call_count == 1  # type: ignore[attr-defined]

{port_ocean-0.30.0 → port_ocean-0.30.2}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -1,8 +1,9 @@
-from typing import cast, Any
-from unittest.mock import AsyncMock, Mock
-from loguru import logger
-import pytest
 from io import StringIO
+from typing import Any, cast
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
+from loguru import logger
 
 from port_ocean.context.ocean import PortOceanContext
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
@@ -10,12 +11,10 @@ from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
 )
 from port_ocean.core.ocean_types import CalculationResult
 from port_ocean.exceptions.core import EntityProcessorException
-from unittest.mock import patch
 
 
 @pytest.mark.asyncio
 class TestJQEntityProcessor:
-
     @pytest.fixture
     def mocked_processor(self, monkeypatch: Any) -> JQEntityProcessor:
         mock_context = AsyncMock()
@@ -33,6 +32,30 @@ class TestJQEntityProcessor:
         result = await mocked_processor._search(data, pattern)
         assert result == "bar"
 
+    async def test_search_with_single_quotes(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"repository": "ocean", "organization": "port"}
+        pattern = ".organization + '/' + .repository"
+        result = await mocked_processor._search(data, pattern)
+        assert result == "port/ocean"
+
+    async def test_search_with_single_quotes_in_the_end(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"organization": "port"}
+        pattern = ".organization + '/'"
+        result = await mocked_processor._search(data, pattern)
+        assert result == "port/"
+
+    async def test_search_with_single_quotes_in_the_start(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"organization": "port"}
+        pattern = "'/' + .organization"
+        result = await mocked_processor._search(data, pattern)
+        assert result == "/port"
+
     async def test_search_as_bool(self, mocked_processor: JQEntityProcessor) -> None:
         data = {"foo": True}
         pattern = ".foo"

{port_ocean-0.30.0 → port_ocean-0.30.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.30.0"
+version = "0.30.2"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"