port-ocean 0.9.2.dev2__py3-none-any.whl → 0.9.4__py3-none-any.whl

port_ocean/config/settings.py

@@ -62,6 +62,7 @@ class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
 
 
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
+    allow_environment_variables_jq_access: bool = True
     initialize_port_resources: bool = True
     scheduled_resync_interval: int | None = None
     client_timeout: int = 30

port_ocean/core/defaults/initialize.py

@@ -71,7 +71,7 @@ async def _initialize_required_integration_settings(
                 integration_config.event_listener.to_request(),
                 port_app_config=default_mapping,
             )
-        elif not integration["config"]:
+        elif not integration.get("config"):
             logger.info(
                 "Encountered that the integration's mapping is empty, Initializing to default mapping"
             )

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -4,7 +4,6 @@ from asyncio import Task
 from dataclasses import dataclass, field
 from functools import lru_cache
 from typing import Any, Optional
-from uuid import uuid4
 
 import pyjq as jq  # type: ignore
 from loguru import logger
@@ -45,62 +44,31 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     @lru_cache
     def _compile(self, pattern: str) -> Any:
+        if not ocean.config.allow_environment_variables_jq_access:
+            pattern = "def env: {}; {} as $ENV | " + pattern
         return jq.compile(pattern)
 
     async def _search(self, data: dict[str, Any], pattern: str) -> Any:
-        identifier = str(uuid4())
         try:
             loop = asyncio.get_event_loop()
-            logger.info(
-                f"__COMPILING__ jq execution with pattern: {pattern}. trace-id: {identifier}"
-            )
             compiled_pattern = self._compile(pattern)
             first_value_callable = functools.partial(compiled_pattern.first, data)
-            logger.info(
-                f"__STARTING__ jq execution with pattern: {pattern} and data: {data}. trace-id: {identifier}"
-            )
-            result = await loop.run_in_executor(None, first_value_callable)
-            logger.info(
-                f"__FINISHED__ jq execution with pattern: {pattern}. trace-id: {identifier} \n"
-                f"Result {result}"
-            )
-            return result
-        except Exception as e:
-            logger.error(
-                f"__FAILED__ jq execution with pattern: {pattern}. trace-id: {identifier} \n"
-                f"Error {e}"
+            return await loop.run_in_executor(None, first_value_callable)
+        except Exception as exc:
+            logger.debug(
+                f"Failed to search for pattern {pattern} in data {data}, {exc}"
             )
             return None
 
     async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
-        identifier = str(uuid4())
-        try:
-            logger.info(
-                f"__COMPILING_BOOL__ jq execution with pattern: {pattern}. trace-id: {identifier}"
-            )
-            compiled_pattern = self._compile(pattern)
-            first_value_callable = functools.partial(compiled_pattern.first, data)
-            logger.info(
-                f"__STARTING_BOOL__ jq execution with pattern: {pattern} and data: {data}. trace-id: {identifier}"
-            )
-            value = await loop.run_in_executor(None, first_value_callable)
-            logger.info(
-                f"__FINISHED_BOOL__ jq execution with pattern: {pattern}. trace-id: {identifier} \n"
-                f"Result {value}"
-            )
-        except Exception as e:
-            logger.error(
-                f"__FAILED_BOOL__ jq execution with pattern: {pattern}. trace-id: {identifier} \n"
-                f"Error {e}"
-            )
-            raise
+        compiled_pattern = self._compile(pattern)
+        first_value_callable = functools.partial(compiled_pattern.first, data)
+        value = await loop.run_in_executor(None, first_value_callable)
 
         if isinstance(value, bool):
             return value
-        logger.error(
-            f"__FAILED_BOOL__ jq execution with pattern: {pattern}. Invalid type got {type(value)} instead. trace-id: {identifier}"
-        )
+
         raise EntityProcessorException(
             f"Expected boolean value, got {type(value)} instead"
         )
@@ -244,6 +212,15 @@ class JQEntityProcessor(BaseEntityProcessor):
                         examples_to_send.append(result.raw_data)
                 else:
                     failed_entities.append(parsed_entity)
+        if (
+            not calculated_entities_results
+            and raw_results
+            and send_raw_data_examples_amount > 0
+        ):
+            logger.warning(
+                f"No entities were parsed from {len(raw_results)} raw results, sending raw data examples"
+            )
+            examples_to_send = raw_results[:send_raw_data_examples_amount]
 
         await self._send_examples(examples_to_send, mapping.kind)
 

port_ocean/core/handlers/port_app_config/base.py

@@ -76,6 +76,7 @@ class BasePortAppConfig(BaseHandler):
                 logger.error(
                     "Invalid port app config found. Please check that the integration has been configured correctly."
                 )
+                logger.warning(f"Invalid port app config: {raw_config}")
                 raise
 
         event.port_app_config = self._app_config_cache.port_app_config

port_ocean/core/integrations/mixins/sync_raw.py

@@ -90,7 +90,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
     ) -> tuple[RESYNC_RESULT, list[RAW_RESULT | Exception]]:
         tasks = []
         results = []
-
         for task in fns:
             if inspect.isasyncgenfunction(task):
                 results.append(resync_generator_wrapper(task, resource_config.kind))
@@ -415,7 +414,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             app_config = await self.port_app_config_handler.get_port_app_config(
                 use_cache=False
             )
-
+            logger.info(f"Resync will use the following mappings: {app_config.dict()}")
             try:
                 entities_at_port = await ocean.port_client.search_entities(
                     user_agent_type

{port_ocean-0.9.2.dev2.dist-info → port_ocean-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.9.2.dev2
+Version: 0.9.4
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.9.2.dev2.dist-info → port_ocean-0.9.4.dist-info}/RECORD

@@ -48,7 +48,7 @@ port_ocean/clients/port/utils.py,sha256=O9mBu6zp4TfpS4SQ3qCPpn9ZVyYF8GKnji4UnYhM
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=J_FUvlo0D4FbEbr4mgJsbGB5JjW9I7zMwfekj6T30KY,6389
 port_ocean/config/dynamic.py,sha256=qOFkRoJsn_BW7581omi_AoMxoHqasf_foxDQ_G11_SI,2030
-port_ocean/config/settings.py,sha256=qGdJUvRTpYv8EOFFlr7xiKXrrA4v9nuhg5EuDevNfqY,3128
+port_ocean/config/settings.py,sha256=Hm2glSmLPtBR2EUtnyOnBmP5rPpbK5d67jfzuPe9gJg,3183
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,7 +59,7 @@ port_ocean/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/core/defaults/__init__.py,sha256=8qCZg8n06WAdMu9s_FiRtDYLGPGHbOuS60vapeUoAks,142
 port_ocean/core/defaults/clean.py,sha256=S3UAfca-oU89WJKIB4OgGjGjPr0vxBQ2aRZsLTZhQ04,2185
 port_ocean/core/defaults/common.py,sha256=QnLFTkT3yIWIRtLQb7fUvvfe5AfInYJy0q5LjlzHkOw,3553
-port_ocean/core/defaults/initialize.py,sha256=FXaxDWHBQtRWUKAKNivrLYeV_YIXWHW8TjjMufXjp38,8861
+port_ocean/core/defaults/initialize.py,sha256=DSY7kbSHQxs6-4ZAK3QYw-kVRsYq5txX7mUjkUB20Qw,8865
 port_ocean/core/event_listener/__init__.py,sha256=mzJ33wRq0kh60fpVdOHVmvMTUQIvz3vxmifyBgwDn0E,889
 port_ocean/core/event_listener/base.py,sha256=4RujgPz4VfDFlviu4qLGJFnJougSCL-Ewf0rfTQfwgc,1133
 port_ocean/core/event_listener/factory.py,sha256=AYYfSHPAF7P5H-uQECXT0JVJjKDHrYkWJJBSL4mGkg8,3697
@@ -77,10 +77,10 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=udR0w5TstTOS5xOfTjAZIEdldn4xr6Oyb3DylatYX3Q,1869
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=qiho974oL73rM8VTA0pqNvqF7UfqoifBE-2F_7yRFNY,9407
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=_nQtlY0ZcCDf_Mpq4qcG7vVQ8LJS6DfkFMv-NHyFiJI,8421
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
-port_ocean/core/handlers/port_app_config/base.py,sha256=oufdNLzUmEgJY5PgIz75zDlowWrjA-y8WR4UnM58E58,2897
+port_ocean/core/handlers/port_app_config/base.py,sha256=-n6b9oLjOA87YqxZv-r7paXxI8R86s44N_g56BEs56A,2970
 port_ocean/core/handlers/port_app_config/models.py,sha256=4dw6HbgjMG3advpN3x6XF35xsgScnWm0KKTERG4CYZ8,2201
 port_ocean/core/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/core/integrations/base.py,sha256=KHsRYFZ38ff3AkkqIQu45883ovgKOJn_fZfnTNy7HWY,2952
@@ -88,7 +88,7 @@ port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4
 port_ocean/core/integrations/mixins/events.py,sha256=Ddfx2L4FpghV38waF8OfVeOV0bHBxNIgjU-q5ffillI,2341
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/sync.py,sha256=TKqRytxXONVhuCo3CB3rDvWNbITnZz33TYTDs3SWWVk,3880
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=yFORznT6TXX7SXP1sSoHAjLxmjDzXIr3Vu-plbhLBoA,18017
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=qgkxlh6_comyPNZ9p52ZhM9I7YQOwaEpn4kR0j20vbw,18103
 port_ocean/core/integrations/mixins/utils.py,sha256=7y1rGETZIjOQadyIjFJXIHKkQFKx_SwiP-TrAIsyyLY,2303
 port_ocean/core/models.py,sha256=4bDvMeydlSpG0YSXjE1Bo2AcZC0aQ2zqFhpsOvZat_A,1190
 port_ocean/core/ocean_types.py,sha256=3_d8-n626f1kWLQ_Jxw194LEyrOVupz05qs_Y1pvB-A,990
@@ -122,8 +122,8 @@ port_ocean/utils/queue_utils.py,sha256=KWWl8YVnG-glcfIHhM6nefY-2sou_C6DVP1VynQwz
 port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,3231
 port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.9.2.dev2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.9.2.dev2.dist-info/METADATA,sha256=RppaJO-SJKGJKRnLueBlzeFx0eB5ickpxsAdyM_AgCA,6566
-port_ocean-0.9.2.dev2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.9.2.dev2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.9.2.dev2.dist-info/RECORD,,
+port_ocean-0.9.4.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.9.4.dist-info/METADATA,sha256=qHdvh3XX8EIrWRn6-Yu8_6hyEL6p0cNHzo0R37tPGkU,6561
+port_ocean-0.9.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.9.4.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.9.4.dist-info/RECORD,,