port-ocean 0.16.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -34,6 +34,7 @@ class MappedEntity:
     entity: dict[str, Any] = field(default_factory=dict)
     did_entity_pass_selector: bool = False
     raw_data: Optional[dict[str, Any]] = None
+    misconfigurations: dict[str, str] = field(default_factory=dict)
 
 
 class JQEntityProcessor(BaseEntityProcessor):
@@ -95,21 +96,37 @@ class JQEntityProcessor(BaseEntityProcessor):
         )
 
     async def _search_as_object(
-        self, data: dict[str, Any], obj: dict[str, Any]
+        self,
+        data: dict[str, Any],
+        obj: dict[str, Any],
+        misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
+        """
+        Identify and extract the relevant value for the chosen key and populate it into the entity
+        :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+            if the data is a dict, we will recursively call this function again.
+        :param obj: the key that we want its value to be mapped into our entity.
+        :param misconfigurations: due to the recursive nature of this function,
+            we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+            we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+        :return: Mapped object with found value.
+        """
+
         search_tasks: dict[
             str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
         ] = {}
         for key, value in obj.items():
             if isinstance(value, list):
                 search_tasks[key] = [
-                    asyncio.create_task(self._search_as_object(data, obj))
+                    asyncio.create_task(
+                        self._search_as_object(data, obj, misconfigurations)
+                    )
                     for obj in value
                 ]
 
             elif isinstance(value, dict):
                 search_tasks[key] = asyncio.create_task(
-                    self._search_as_object(data, value)
+                    self._search_as_object(data, value, misconfigurations)
                 )
             else:
                 search_tasks[key] = asyncio.create_task(self._search(data, value))
@@ -118,12 +135,20 @@ class JQEntityProcessor(BaseEntityProcessor):
         for key, task in search_tasks.items():
             try:
                 if isinstance(task, list):
-                    result[key] = [await task for task in task]
+                    result_list = []
+                    for task in task:
+                        task_result = await task
+                        if task_result is None and misconfigurations is not None:
+                            misconfigurations[key] = obj[key]
+                        result_list.append(task_result)
+                    result[key] = result_list
                 else:
-                    result[key] = await task
+                    task_result = await task
+                    if task_result is None and misconfigurations is not None:
+                        misconfigurations[key] = obj[key]
+                    result[key] = task_result
             except Exception:
                 result[key] = None
-
         return result
 
     async def _get_mapped_entity(
@@ -135,11 +160,15 @@ class JQEntityProcessor(BaseEntityProcessor):
     ) -> MappedEntity:
         should_run = await self._search_as_bool(data, selector_query)
         if parse_all or should_run:
-            mapped_entity = await self._search_as_object(data, raw_entity_mappings)
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
+            )
             return MappedEntity(
                 mapped_entity,
                 did_entity_pass_selector=should_run,
                 raw_data=data if should_run else None,
+                misconfigurations=misconfigurations,
             )
 
         return MappedEntity()
@@ -221,7 +250,11 @@ class JQEntityProcessor(BaseEntityProcessor):
         passed_entities = []
         failed_entities = []
         examples_to_send: list[dict[str, Any]] = []
+        entity_misconfigurations: dict[str, str] = {}
+        missing_required_fields: bool = False
         for result in calculated_entities_results:
+            if len(result.misconfigurations) > 0:
+                entity_misconfigurations |= result.misconfigurations
             if result.entity.get("identifier") and result.entity.get("blueprint"):
                 parsed_entity = Entity.parse_obj(result.entity)
                 if result.did_entity_pass_selector:
@@ -233,6 +266,12 @@ class JQEntityProcessor(BaseEntityProcessor):
                         examples_to_send.append(result.raw_data)
                 else:
                     failed_entities.append(parsed_entity)
+            else:
+                missing_required_fields = True
+        if len(entity_misconfigurations) > 0:
+            logger.info(
+                f"The mapping resulted with invalid values for{" identifier, blueprint," if missing_required_fields else " "} properties. Mapping result: {entity_misconfigurations}"
+            )
         if (
             not calculated_entities_results
             and raw_results
@@ -248,4 +287,5 @@ class JQEntityProcessor(BaseEntityProcessor):
         return CalculationResult(
             EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
             errors,
+            misonfigured_entity_keys=entity_misconfigurations,
         )

port_ocean/core/integrations/mixins/sync_raw.py

@@ -185,7 +185,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         send_raw_data_examples_amount = (
             SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
-        all_entities, register_errors = await self._register_resource_raw(
+        all_entities, register_errors,_ = await self._register_resource_raw(
             resource_config,
             raw_results,
             user_agent_type,
@@ -202,7 +202,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             0, send_raw_data_examples_amount - len(passed_entities)
                         )
 
-                    entities, register_errors = await self._register_resource_raw(
+                    entities, register_errors,_ = await self._register_resource_raw(
                         resource_config,
                         items,
                         user_agent_type,

port_ocean/core/ocean_types.py

@@ -1,5 +1,13 @@
-from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable, NamedTuple
-
+from typing import (
+    TypedDict,
+    Any,
+    AsyncIterator,
+    Callable,
+    Awaitable,
+    NamedTuple,
+)
+
+from dataclasses import field
 from port_ocean.core.models import Entity
 
 RAW_ITEM = dict[Any, Any]
@@ -30,6 +38,7 @@ class EntitySelectorDiff(NamedTuple):
 class CalculationResult(NamedTuple):
     entity_selector_diff: EntitySelectorDiff
     errors: list[Exception]
+    misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
 
 
 class IntegrationEventsCallbacks(TypedDict):

port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -269,3 +269,37 @@ class TestJQEntityProcessor:
             assert len(result.entity_selector_diff.passed) == 1
             assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
             assert not result.errors
+
+    async def test_parse_items_wrong_mapping(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "title": ".foo",
+            "identifier": ".ark",
+            "blueprint": ".baz",
+            "properties": {
+                "description": ".bazbar",
+                "url": ".foobar",
+                "defaultBranch": ".bar.baz",
+            },
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = "true"
+        raw_results = [
+            {
+                "foo": "bar",
+                "baz": "bazbar",
+                "bar": {"foobar": "barfoo", "baz": "barbaz"},
+            },
+            {"foo": "bar", "baz": "bazbar", "bar": {"foobar": "foobar"}},
+        ]
+        result = await mocked_processor._parse_items(mapping, raw_results)
+        assert len(result.misonfigured_entity_keys) > 0
+        assert len(result.misonfigured_entity_keys) == 4
+        assert result.misonfigured_entity_keys == {
+            "identifier": ".ark",
+            "description": ".bazbar",
+            "url": ".foobar",
+            "defaultBranch": ".bar.baz",
+        }

{port_ocean-0.16.0.dist-info → port_ocean-0.16.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.16.0
+Version: 0.16.1
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.16.0.dist-info → port_ocean-0.16.1.dist-info}/RECORD

@@ -87,7 +87,7 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=udR0w5TstTOS5xOfTjAZIEdldn4xr6Oyb3DylatYX3Q,1869
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=X0aYDqgvuZG813JM1qjWpK-V9ASgctTRNgz02wJ4vMY,8887
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=X-up0HVdE8pkITxzvB1BC7W8Oq0C14WbT3WqV7p-wJc,11129
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
 port_ocean/core/handlers/port_app_config/base.py,sha256=4Nxt2g8voEIHJ4Y1Km5NJcaG2iSbCklw5P8-Kus7Y9k,3007
@@ -100,10 +100,10 @@ port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4
 port_ocean/core/integrations/mixins/events.py,sha256=Ddfx2L4FpghV38waF8OfVeOV0bHBxNIgjU-q5ffillI,2341
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/sync.py,sha256=B9fEs8faaYLLikH9GBjE_E61vo0bQDjIGQsQ1SRXOlA,3931
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=FjmYIP-7fqQY9tlqN9tNyHwdN81Dn4sqxVMI6JJbRYE,20275
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=Wir4aTSCkIvG6Ny9Eo0Xf55OkSbh_6wHfNSaCffAKJQ,20279
 port_ocean/core/integrations/mixins/utils.py,sha256=oN4Okz6xlaefpid1_Pud8HPSw9BwwjRohyNsknq-Myg,2309
 port_ocean/core/models.py,sha256=O8nOKc4ORZz9tS5s6y5YgGLEBroXpvSPDqKuz48uKvs,1965
-port_ocean/core/ocean_types.py,sha256=3_d8-n626f1kWLQ_Jxw194LEyrOVupz05qs_Y1pvB-A,990
+port_ocean/core/ocean_types.py,sha256=j_-or1VxDy22whLLxwxgzIsE4wAhFLH19Xff9l4oJA8,1124
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
 port_ocean/core/utils/utils.py,sha256=QSRuF9wlhbOw6cELlDlek_UIX6ciIuKWml8QhBmHU_k,3703
 port_ocean/debug_cli.py,sha256=gHrv-Ey3cImKOcGZpjoHlo4pa_zfmyOl6TUM4o9VtcA,96
@@ -131,7 +131,7 @@ port_ocean/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 port_ocean/tests/clients/port/mixins/test_entities.py,sha256=A9myrnkLhKSQrnOLv1Zz2wiOVSxW65Q9RIUIRbn_V7w,1586
 port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,101
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
-port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=Yv03P-LDcJCKZ21exiTFrcT1eu0zn6Z954dilxrb52Y,10842
+port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=C2nLgapTdXRrzP5B4xBuHcc14L-NztFpxLIv8Iuv6Gg,12046
 port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=RPrbw4Zs6bmhL9zMQviq7-qMfgP5_4nJDkfZiAukK-g,15782
 port_ocean/tests/core/test_utils.py,sha256=Z3kdhb5V7Svhcyy3EansdTpgHL36TL6erNtU-OPwAcI,2647
 port_ocean/tests/core/utils/test_entity_topological_sorter.py,sha256=zuq5WSPy_88PemG3mOUIHTxWMR_js1R7tOzUYlgBd68,3447
@@ -156,8 +156,8 @@ port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,32
 port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.16.0.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.16.0.dist-info/METADATA,sha256=IonSZuh115163Dk9ISkmRGEHSnavV_0i58YgSuz2mek,6673
-port_ocean-0.16.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.16.0.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.16.0.dist-info/RECORD,,
+port_ocean-0.16.1.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.16.1.dist-info/METADATA,sha256=6Zbwkb10_nBvr-Yan20BoYf42oD77C0je7Ft2UXmZHs,6673
+port_ocean-0.16.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.16.1.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.16.1.dist-info/RECORD,,