port-ocean 0.12.2.dev5__tar.gz → 0.12.2.dev7__tar.gz

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.2.dev5
+Version: 0.12.2.dev7
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/clients/port/mixins/entities.py

@@ -56,33 +56,33 @@ class EntityClientMixin:
                 f"entity: {entity.identifier} of "
                 f"blueprint: {entity.blueprint}"
             )
-        # handle_status_code(response, should_raise)
-        # result = response.json()
-        #
-        # result_entity = (
-        #     Entity.parse_obj(result["entity"]) if result.get("entity") else entity
-        # )
-        #
-        # # Happens when upsert fails and search identifier is defined.
-        # # We return None to ignore the entity later in the delete process
-        # if result_entity.is_using_search_identifier:
-        #     return None
-        #
-        # # In order to save memory we'll keep only the identifier, blueprint and relations of the
-        # # upserted entity result for later calculations
-        # reduced_entity = Entity(
-        #     identifier=result_entity.identifier, blueprint=result_entity.blueprint
-        # )
-        #
-        # # Turning dict typed relations (raw search relations) is required
-        # # for us to be able to successfully calculate the participation related entities
-        # # and ignore the ones that don't as they weren't upserted
-        # reduced_entity.relations = {
-        #     key: None if isinstance(relation, dict) else relation
-        #     for key, relation in result_entity.relations.items()
-        # }
-
-        return None
+        handle_status_code(response, should_raise)
+        result = response.json()
+
+        result_entity = (
+            Entity.parse_obj(result["entity"]) if result.get("entity") else entity
+        )
+
+        # Happens when upsert fails and search identifier is defined.
+        # We return None to ignore the entity later in the delete process
+        if result_entity.is_using_search_identifier:
+            return None
+
+        # In order to save memory we'll keep only the identifier, blueprint and relations of the
+        # upserted entity result for later calculations
+        reduced_entity = Entity(
+            identifier=result_entity.identifier, blueprint=result_entity.blueprint
+        )
+
+        # Turning dict typed relations (raw search relations) is required
+        # for us to be able to successfully calculate the participation related entities
+        # and ignore the ones that don't as they weren't upserted
+        reduced_entity.relations = {
+            key: None if isinstance(relation, dict) else relation
+            for key, relation in result_entity.relations.items()
+        }
+
+        return reduced_entity
 
     async def batch_upsert_entities(
         self,
@@ -91,7 +91,7 @@ class EntityClientMixin:
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
     ) -> list[Entity]:
-        await asyncio.gather(
+        modified_entities_results = await asyncio.gather(
             *(
                 self.upsert_entity(
                     entity,
@@ -103,18 +103,17 @@ class EntityClientMixin:
             ),
             return_exceptions=True,
         )
-        return []
-        # entity_results = [
-        #     entity for entity in modified_entities_results if isinstance(entity, Entity)
-        # ]
-        # if not should_raise:
-        #     return entity_results
-        #
-        # for entity_result in modified_entities_results:
-        #     if isinstance(entity_result, Exception):
-        #         raise entity_result
-        #
-        # return entity_results
+        entity_results = [
+            entity for entity in modified_entities_results if isinstance(entity, Entity)
+        ]
+        if not should_raise:
+            return entity_results
+
+        for entity_result in modified_entities_results:
+            if isinstance(entity_result, Exception):
+                raise entity_result
+
+        return entity_results
 
     async def delete_entity(
         self,

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/core/event_listener/base.py

@@ -1,14 +1,12 @@
-import asyncio
 from abc import abstractmethod
-from concurrent.futures import ProcessPoolExecutor
 from typing import TypedDict, Callable, Any, Awaitable
 
 from pydantic import Extra
 
 from port_ocean.config.base import BaseOceanModel
+from port_ocean.utils.signal import signal_handler
 from port_ocean.context.ocean import ocean
 from port_ocean.utils.misc import IntegrationStateStatus
-from port_ocean.utils.signal import signal_handler
 
 
 class EventListenerEvents(TypedDict):
@@ -69,14 +67,7 @@ class BaseEventListener:
         """
         await self._before_resync()
         try:
-            loop = asyncio.get_event_loop()
-            with ProcessPoolExecutor() as executor:
-                e = executor.submit(
-                    lambda: asyncio.run_coroutine_threadsafe(
-                        self.events["on_resync"](resync_args), loop
-                    )
-                )
-                signal_handler.register(e.cancel)
+            await self.events["on_resync"](resync_args)
             await self._after_resync()
         except Exception as e:
             await self._on_resync_failure(e)

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -97,38 +97,37 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         self, entities: list[Entity], user_agent_type: UserAgentType
     ) -> list[Entity]:
         logger.info(f"Upserting {len(entities)} entities")
-        # modified_entities: list[Entity] = []
-        # if event.port_app_config.create_missing_related_entities:
-        await self.context.port_client.batch_upsert_entities(
-            entities,
-            event.port_app_config.get_port_request_options(),
-            user_agent_type,
-            should_raise=False,
-        )
-        # else:
-        #     entities_with_search_identifier: list[Entity] = []
-        #     entities_without_search_identifier: list[Entity] = []
-        #     for entity in entities:
-        #         if entity.is_using_search_identifier:
-        #             entities_with_search_identifier.append(entity)
-        #         else:
-        #             entities_without_search_identifier.append(entity)
-        #
-        #     ordered_created_entities = reversed(
-        #         entities_with_search_identifier
-        #         + order_by_entities_dependencies(entities_without_search_identifier)
-        #     )
-        #     for entity in ordered_created_entities:
-        #         upsertedEntity = await self.context.port_client.upsert_entity(
-        #             entity,
-        #             event.port_app_config.get_port_request_options(),
-        #             user_agent_type,
-        #             should_raise=False,
-        #         )
-        #         if upsertedEntity:
-        #             modified_entities.append(upsertedEntity)
-        # return modified_entities
-        return []
+        modified_entities: list[Entity] = []
+        if event.port_app_config.create_missing_related_entities:
+            modified_entities = await self.context.port_client.batch_upsert_entities(
+                entities,
+                event.port_app_config.get_port_request_options(),
+                user_agent_type,
+                should_raise=False,
+            )
+        else:
+            entities_with_search_identifier: list[Entity] = []
+            entities_without_search_identifier: list[Entity] = []
+            for entity in entities:
+                if entity.is_using_search_identifier:
+                    entities_with_search_identifier.append(entity)
+                else:
+                    entities_without_search_identifier.append(entity)
+
+            ordered_created_entities = reversed(
+                entities_with_search_identifier
+                + order_by_entities_dependencies(entities_without_search_identifier)
+            )
+            for entity in ordered_created_entities:
+                upsertedEntity = await self.context.port_client.upsert_entity(
+                    entity,
+                    event.port_app_config.get_port_request_options(),
+                    user_agent_type,
+                    should_raise=False,
+                )
+                if upsertedEntity:
+                    modified_entities.append(upsertedEntity)
+        return modified_entities
 
     async def delete(
         self, entities: list[Entity], user_agent_type: UserAgentType

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -3,6 +3,7 @@ import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -139,11 +140,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         objects_diff = await self._calculate_raw(
             [(resource, results)], parse_all, send_raw_data_examples_amount
         )
-        await self.entities_state_applier.upsert(
+        modified_objects = await self.entities_state_applier.upsert(
             objects_diff[0].entity_selector_diff.passed, user_agent_type
         )
         return CalculationResult(
-            objects_diff[0].entity_selector_diff._replace(passed=objects_diff[0].entity_selector_diff.passed),
+            objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
             errors=objects_diff[0].errors,
         )
 
@@ -172,48 +173,49 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
     ) -> tuple[list[Entity], list[Exception]]:
         results, errors = await self._get_resource_raw_results(resource_config)
-        async_generators: list[ASYNC_GENERATOR_RESYNC_TYPE] = []
-        raw_results: RAW_RESULT = []
-        for result in results:
-            if isinstance(result, dict):
-                raw_results.append(result)
-            else:
-                async_generators.append(result)
-
-        send_raw_data_examples_amount = (
-            SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
-        )
-        all_entities, register_errors = await self._register_resource_raw(
-            resource_config,
-            raw_results,
-            user_agent_type,
-            send_raw_data_examples_amount=send_raw_data_examples_amount,
-        )
-        errors.extend(register_errors)
-        passed_entities = list(all_entities.passed)
-
-        for generator in async_generators:
-            try:
-                async for items in generator:
-                    if send_raw_data_examples_amount > 0:
-                        send_raw_data_examples_amount = max(
-                            0, send_raw_data_examples_amount - len(passed_entities)
-                        )
-
-                    entities, register_errors = await self._register_resource_raw(
-                        resource_config,
-                        items,
-                        user_agent_type,
-                        send_raw_data_examples_amount=send_raw_data_examples_amount,
-                    )
-                    errors.extend(register_errors)
-                    passed_entities.extend(entities.passed)
-            except* OceanAbortException as error:
-                errors.append(error)
-
-        logger.info(
-            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
-        )
+        passed_entities = []
+        # async_generators: list[ASYNC_GENERATOR_RESYNC_TYPE] = []
+        # raw_results: RAW_RESULT = []
+        # for result in results:
+        #     if isinstance(result, dict):
+        #         raw_results.append(result)
+        #     else:
+        #         async_generators.append(result)
+        #
+        # send_raw_data_examples_amount = (
+        #     SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
+        # )
+        # all_entities, register_errors = await self._register_resource_raw(
+        #     resource_config,
+        #     raw_results,
+        #     user_agent_type,
+        #     send_raw_data_examples_amount=send_raw_data_examples_amount,
+        # )
+        # errors.extend(register_errors)
+        # passed_entities = list(all_entities.passed)
+        #
+        # for generator in async_generators:
+        #     try:
+        #         async for items in generator:
+        #             if send_raw_data_examples_amount > 0:
+        #                 send_raw_data_examples_amount = max(
+        #                     0, send_raw_data_examples_amount - len(passed_entities)
+        #                 )
+        #
+        #             entities, register_errors = await self._register_resource_raw(
+        #                 resource_config,
+        #                 items,
+        #                 user_agent_type,
+        #                 send_raw_data_examples_amount=send_raw_data_examples_amount,
+        #             )
+        #             errors.extend(register_errors)
+        #             passed_entities.extend(entities.passed)
+        #     except* OceanAbortException as error:
+        #         errors.append(error)
+        #
+        # logger.info(
+        #     f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
+        # )
         return passed_entities, errors
 
     async def register_raw(

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/log/handlers.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
 import sys
+import threading
 import time
-from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from logging.handlers import MemoryHandler
 from typing import Any
@@ -65,19 +65,18 @@ class HTTPMemoryHandler(MemoryHandler):
         if self.ocean is None or not self.buffer:
             return
 
+        def _wrap_event_loop(_ocean: Ocean, logs_to_send: list[dict[str, Any]]) -> None:
+            loop = asyncio.new_event_loop()
+            loop.run_until_complete(self.send_logs(_ocean, logs_to_send))
+            loop.close()
+
         self.acquire()
         logs = list(self._serialized_buffer)
         if logs:
             self.buffer.clear()
             self._serialized_buffer.clear()
             self.last_flush_time = time.time()
-            loop = asyncio.new_event_loop()
-            with ProcessPoolExecutor() as executor:
-                executor.submit(
-                    lambda: asyncio.run_coroutine_threadsafe(
-                        self.send_logs(self.ocean, logs), loop
-                    )
-                )
+            threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs)).start()
         self.release()
 
     async def send_logs(

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/port_ocean/ocean.py

@@ -1,6 +1,6 @@
 import asyncio
 import sys
-from concurrent.futures import ProcessPoolExecutor
+import threading
 from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
@@ -9,6 +9,8 @@ from loguru import logger
 from pydantic import BaseModel
 from starlette.types import Scope, Receive, Send
 
+from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
+from port_ocean.core.models import Runtime
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import (
     IntegrationConfiguration,
@@ -18,15 +20,13 @@ from port_ocean.context.ocean import (
     ocean,
     initialize_port_ocean_context,
 )
-from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.integrations.base import BaseIntegration
-from port_ocean.core.models import Runtime
 from port_ocean.log.sensetive import sensitive_log_filter
 from port_ocean.middlewares import request_handler
-from port_ocean.utils.misc import IntegrationStateStatus
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
+from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class Ocean:
@@ -92,13 +92,6 @@ class Ocean:
                 )
                 raise e
 
-        def pool_executor_wrapper(event_loop) -> None:
-            with ProcessPoolExecutor() as executor:
-                e = executor.submit(lambda: asyncio.run_coroutine_threadsafe(
-                    execute_resync_all(), event_loop
-                ))
-                signal_handler.register(e.cancel)
-
         interval = self.config.scheduled_resync_interval
         loop = asyncio.get_event_loop()
         if interval is not None:
@@ -110,7 +103,13 @@ class Ocean:
                 seconds=interval * 60,
                 # Not running the resync immediately because the event listener should run resync on startup
                 wait_first=True,
-            )(lambda _: pool_executor_wrapper(loop))
+            )(
+                lambda: threading.Thread(
+                    target=lambda: asyncio.run_coroutine_threadsafe(
+                        execute_resync_all(), loop
+                    )
+                ).start()
+            )
             await repeated_function()
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:

{port_ocean-0.12.2.dev5 → port_ocean-0.12.2.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.12.2-dev05"
+version = "0.12.2-dev07"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"