port-ocean 0.5.5__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py

@@ -15,6 +15,21 @@ async def on_resync(kind: str) -> list[dict[Any, Any]]:
     #     return [{"some_project_key": "someProjectValue", ...}]
     # if kind == "issues":
     #     return [{"some_issue_key": "someIssueValue", ...}]
+
+    # Initial stub to show complete flow, replace this with your own logic
+    if kind == "{{ cookiecutter.integration_slug }}-example-kind":
+        return [
+            {
+                "my_custom_id": f"id_{x}",
+                "my_custom_text": f"very long text with {x} in it",
+                "my_special_score": x * 32 % 3,
+                "my_component": f"component-{x}",
+                "my_service": f"service-{x %2}",
+                "my_enum": "VALID" if x % 2 == 0 else "FAILED",
+            }
+            for x in range(25)
+        ]
+
     return []
 
 
@@ -38,4 +53,4 @@ async def on_resync(kind: str) -> list[dict[Any, Any]]:
 async def on_start() -> None:
     # Something to do when the integration starts
     # For example create a client to query 3rd party services - GitHub, Jira, etc...
-    print("Starting integration")
+    print("Starting {{ cookiecutter.integration_slug }} integration")

port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml

@@ -1,25 +1,31 @@
 [tool.poetry]
-name = "{{cookiecutter.integration_name}}"
-version = "0.1.0"
+name = "{{cookiecutter.integration_slug}}"
+version = "0.1.0-beta"
 description = "{{cookiecutter.integration_short_description}}"
 authors = ["{{cookiecutter.full_name}} <{{cookiecutter.email}}>"]
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = "^3.12"
 port_ocean = { version = "^{% version %}", extras = ["cli"] }
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.2"
-black = "^23.3.0"
+# Uncomment this if you want to debug the ocean core together with your integration
+# port_ocean = { path = '../../', develop = true, extras = ['all'] }
+black = "^24.4.2"
 mypy = "^1.3.0"
-ruff = "^0.0.278"
-pylint = "^2.17.4"
+pylint = ">=2.17.4,<4.0.0"
+pytest = ">=8.2,<9.0"
+pytest-asyncio = ">=0.24.0"
+pytest-httpx = ">=0.30.0"
+pytest-xdist = "^3.6.1"
+ruff = "^0.6.3"
 towncrier = "^23.6.0"
 
 [tool.towncrier]
 directory = "changelog"
 filename = "CHANGELOG.md"
-package = "port_ocean"
+title_format = "## {version} ({project_date})"
+underlines = [""]
 
   [[tool.towncrier.type]]
   directory = "breaking"
@@ -52,8 +58,8 @@ package = "port_ocean"
   showcontent = true
 
 [build-system]
-requires = ["poetry>=0.12"]
-build-backend = "poetry.masonry.api"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
 
 [tool.mypy]
 exclude = [
@@ -100,3 +106,8 @@ exclude = '''
   |\.venv
 )/
 '''
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+addopts = "-vv -n auto ./tests"

port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py

@@ -0,0 +1,2 @@
+def test_example() -> None:
+    assert 1 == 1

port_ocean/clients/port/authentication.py

@@ -1,3 +1,4 @@
+import re
 from typing import Any
 
 import httpx
@@ -13,11 +14,11 @@ class TokenResponse(BaseModel):
     access_token: str = Field(alias="accessToken")
     expires_in: int = Field(alias="expiresIn")
     token_type: str = Field(alias="tokenType")
-    _retrieved_time: int = PrivateAttr(get_time())
+    _retrieved_time: int = PrivateAttr(default_factory=lambda: int(get_time()))
 
     @property
     def expired(self) -> bool:
-        return self._retrieved_time + self.expires_in < get_time()
+        return self._retrieved_time + self.expires_in <= get_time()
 
     @property
     def full_token(self) -> str:
@@ -46,10 +47,16 @@ class PortAuthentication:
 
     async def _get_token(self, client_id: str, client_secret: str) -> TokenResponse:
         logger.info(f"Fetching access token for clientId: {client_id}")
-
+        if self._is_personal_token(client_id):
+            logger.warning(
+                "Integration is using personal credentials, make sure to use machine credentials. "
+                "Usage of personal credentials might impose unexpected integration behavior."
+            )
         credentials = {"clientId": client_id, "clientSecret": client_secret}
         response = await self.client.post(
-            f"{self.api_url}/auth/access_token", json=credentials
+            f"{self.api_url}/auth/access_token",
+            json=credentials,
+            extensions={"retryable": True},
         )
         handle_status_code(response)
         return TokenResponse(**response.json())
@@ -80,3 +87,8 @@ class PortAuthentication:
                 self.client_id, self.client_secret
             )
         return self.last_token_object.full_token
+
+    @staticmethod
+    def _is_personal_token(client_id: str) -> bool:
+        email_regex = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+        return re.match(email_regex, client_id) is not None

port_ocean/clients/port/client.py

@@ -13,6 +13,7 @@ from port_ocean.clients.port.utils import (
     get_internal_http_client,
 )
 from port_ocean.exceptions.clients import KafkaCredentialsNotFound
+from typing import Any
 
 
 class PortClient(
@@ -75,3 +76,19 @@ class PortClient(
         handle_status_code(response)
 
         return response.json()["organization"]["id"]
+
+    async def update_integration_state(
+        self, state: dict[str, Any], should_raise: bool = True, should_log: bool = True
+    ) -> dict[str, Any]:
+        if should_log:
+            logger.debug(f"Updating integration resync state with: {state}")
+        response = await self.client.patch(
+            f"{self.api_url}/integration/{self.integration_identifier}/resync-state",
+            headers=await self.auth.headers(),
+            json=state,
+        )
+        handle_status_code(response, should_raise, should_log)
+        if response.is_success and should_log:
+            logger.info("Integration resync state updated successfully")
+
+        return response.json().get("integration", {})

port_ocean/clients/port/mixins/blueprints.py

@@ -64,7 +64,6 @@ class BlueprintClientMixin:
             f"Deleting blueprint with id: {identifier} with all entities: {delete_entities}"
         )
         headers = await self.auth.headers(user_agent_type)
-        response = None
 
         if not delete_entities:
             response = await self.client.delete(
@@ -83,21 +82,22 @@ class BlueprintClientMixin:
             return response.json().get("migrationId", "")
 
     async def create_action(
-        self, blueprint_identifier: str, action: dict[str, Any]
+        self, action: dict[str, Any], should_log: bool = True
     ) -> None:
         logger.info(f"Creating action: {action}")
         response = await self.client.post(
-            f"{self.auth.api_url}/blueprints/{blueprint_identifier}/actions",
+            f"{self.auth.api_url}/actions",
             json=action,
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
 
     async def create_scorecard(
         self,
         blueprint_identifier: str,
         scorecard: dict[str, Any],
+        should_log: bool = True,
     ) -> None:
         logger.info(f"Creating scorecard: {scorecard}")
         response = await self.client.post(
@@ -106,11 +106,10 @@ class BlueprintClientMixin:
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
 
     async def create_page(
-        self,
-        page: dict[str, Any],
+        self, page: dict[str, Any], should_log: bool = True
     ) -> dict[str, Any]:
         logger.info(f"Creating page: {page}")
         response = await self.client.post(
@@ -119,7 +118,7 @@ class BlueprintClientMixin:
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
         return page
 
     async def delete_page(

port_ocean/clients/port/mixins/entities.py

@@ -1,4 +1,5 @@
 import asyncio
+from typing import Any, Literal
 from urllib.parse import quote_plus
 
 import httpx
@@ -10,7 +11,8 @@ from port_ocean.clients.port.utils import (
     handle_status_code,
     PORT_HTTP_MAX_CONNECTIONS_LIMIT,
 )
-from port_ocean.core.models import Entity
+from port_ocean.core.models import Entity, PortAPIErrorMessage
+from starlette import status
 
 
 class EntityClientMixin:
@@ -28,10 +30,30 @@ class EntityClientMixin:
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> None:
+    ) -> Entity | None | Literal[False]:
+        """
+        This function upserts an entity into Port.
+
+        Usage:
+        ```python
+            upsertedEntity = await self.context.port_client.upsert_entity(
+                            entity,
+                            event.port_app_config.get_port_request_options(),
+                            user_agent_type,
+                            should_raise=False,
+                        )
+        ```
+        :param entity: An Entity to be upserted
+        :param request_options: A dictionary specifying how to upsert the entity
+        :param user_agent_type: a UserAgentType specifying who is preforming the action
+        :param should_raise: A boolean specifying whether the error should be raised or handled silently
+        :return: [Entity] if the upsert occured successfully
+        :return: [None] will be returned if entity is using search identifier
+        :return: [False] will be returned if upsert failed because of unmet dependency
+        """
         validation_only = request_options["validation_only"]
         async with self.semaphore:
-            logger.info(
+            logger.debug(
                 f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
             )
             headers = await self.auth.headers(user_agent_type)
@@ -47,15 +69,50 @@ class EntityClientMixin:
                     ).lower(),
                     "validation_only": str(validation_only).lower(),
                 },
+                extensions={"retryable": True},
             )
-
         if response.is_error:
             logger.error(
                 f"Error {'Validating' if validation_only else 'Upserting'} "
                 f"entity: {entity.identifier} of "
                 f"blueprint: {entity.blueprint}"
             )
+            result = response.json()
+
+            if (
+                response.status_code == status.HTTP_404_NOT_FOUND
+                and not result.get("ok")
+                and result.get("error") == PortAPIErrorMessage.NOT_FOUND.value
+            ):
+                # Return false to differentiate from `result_entity.is_using_search_identifier`
+                return False
         handle_status_code(response, should_raise)
+        result = response.json()
+
+        result_entity = (
+            Entity.parse_obj(result["entity"]) if result.get("entity") else entity
+        )
+
+        # Happens when upsert fails and search identifier is defined.
+        # We return None to ignore the entity later in the delete process
+        if result_entity.is_using_search_identifier:
+            return None
+
+        # In order to save memory we'll keep only the identifier, blueprint and relations of the
+        # upserted entity result for later calculations
+        reduced_entity = Entity(
+            identifier=result_entity.identifier, blueprint=result_entity.blueprint
+        )
+
+        # Turning dict typed relations (raw search relations) is required
+        # for us to be able to successfully calculate the participation related entities
+        # and ignore the ones that don't as they weren't upserted
+        reduced_entity.relations = {
+            key: None if isinstance(relation, dict) else relation
+            for key, relation in result_entity.relations.items()
+        }
+
+        return reduced_entity
 
     async def batch_upsert_entities(
         self,
@@ -63,8 +120,8 @@ class EntityClientMixin:
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> None:
-        await asyncio.gather(
+    ) -> list[Entity]:
+        modified_entities_results = await asyncio.gather(
             *(
                 self.upsert_entity(
                     entity,
@@ -76,6 +133,17 @@ class EntityClientMixin:
             ),
             return_exceptions=True,
         )
+        entity_results = [
+            entity for entity in modified_entities_results if isinstance(entity, Entity)
+        ]
+        if not should_raise:
+            return entity_results
+
+        for entity_result in modified_entities_results:
+            if isinstance(entity_result, Exception):
+                raise entity_result
+
+        return entity_results
 
     async def delete_entity(
         self,
@@ -133,23 +201,10 @@ class EntityClientMixin:
             return_exceptions=True,
         )
 
-    async def validate_entity_exist(self, identifier: str, blueprint: str) -> None:
-        logger.info(f"Validating entity {identifier} of blueprint {blueprint} exists")
-
-        response = await self.client.get(
-            f"{self.auth.api_url}/blueprints/{blueprint}/entities/{identifier}",
-            headers=await self.auth.headers(),
-        )
-        if response.is_error:
-            logger.error(
-                f"Error validating "
-                f"entity: {identifier} of "
-                f"blueprint: {blueprint}"
-            )
-        handle_status_code(response)
-
-    async def search_entities(self, user_agent_type: UserAgentType) -> list[Entity]:
-        query = {
+    async def search_entities(
+        self, user_agent_type: UserAgentType, query: dict[Any, Any] | None = None
+    ) -> list[Entity]:
+        default_query = {
             "combinator": "and",
             "rules": [
                 {
@@ -165,6 +220,11 @@ class EntityClientMixin:
             ],
         }
 
+        if query is None:
+            query = default_query
+        elif query.get("rules"):
+            query["rules"].append(default_query)
+
         logger.info(f"Searching entities with query {query}")
         response = await self.client.post(
             f"{self.auth.api_url}/entities/search",
@@ -174,43 +234,38 @@ class EntityClientMixin:
                 "exclude_calculated_properties": "true",
                 "include": ["blueprint", "identifier"],
             },
+            extensions={"retryable": True},
         )
         handle_status_code(response)
         return [Entity.parse_obj(result) for result in response.json()["entities"]]
 
-    async def search_dependent_entities(self, entity: Entity) -> list[Entity]:
-        body = {
-            "combinator": "and",
-            "rules": [
+    async def search_batch_entities(
+        self, user_agent_type: UserAgentType, entities_to_search: list[Entity]
+    ) -> list[Entity]:
+        search_rules = []
+        for entity in entities_to_search:
+            search_rules.append(
                 {
-                    "operator": "relatedTo",
-                    "blueprint": entity.blueprint,
-                    "value": entity.identifier,
-                    "direction": "downstream",
+                    "combinator": "and",
+                    "rules": [
+                        {
+                            "property": "$identifier",
+                            "operator": "=",
+                            "value": entity.identifier,
+                        },
+                        {
+                            "property": "$blueprint",
+                            "operator": "=",
+                            "value": entity.blueprint,
+                        },
+                    ],
                 }
-            ],
-        }
-
-        logger.info(f"Searching dependent entity with body {body}")
-        response = await self.client.post(
-            f"{self.auth.api_url}/entities/search",
-            headers=await self.auth.headers(),
-            json=body,
-        )
-        handle_status_code(response)
-
-        return [Entity.parse_obj(result) for result in response.json()["entities"]]
+            )
 
-    async def validate_entity_payload(
-        self, entity: Entity, merge: bool, create_missing_related_entities: bool
-    ) -> None:
-        logger.info(f"Validating entity {entity.identifier}")
-        await self.upsert_entity(
-            entity,
+        return await self.search_entities(
+            user_agent_type,
             {
-                "merge": merge,
-                "create_missing_related_entities": create_missing_related_entities,
-                "delete_dependent_entities": False,
-                "validation_only": True,
+                "combinator": "and",
+                "rules": [{"combinator": "or", "rules": search_rules}],
             },
         )

port_ocean/clients/port/mixins/integrations.py

@@ -1,11 +1,11 @@
 from typing import Any, TYPE_CHECKING, Optional, TypedDict
+from urllib.parse import quote_plus
 
 import httpx
 from loguru import logger
-from starlette import status
-
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.utils import handle_status_code
+from port_ocean.log.sensetive import sensitive_log_filter
 
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import PortAppConfig
@@ -42,7 +42,7 @@ class IntegrationClientMixin:
     ) -> dict[str, Any]:
         response = await self._get_current_integration()
         handle_status_code(response, should_raise, should_log)
-        return response.json()["integration"]
+        return response.json().get("integration", {})
 
     async def get_log_attributes(self) -> LogAttributes:
         if self._log_attributes is None:
@@ -55,7 +55,7 @@ class IntegrationClientMixin:
         _type: str,
         changelog_destination: dict[str, Any],
         port_app_config: Optional["PortAppConfig"] = None,
-    ) -> None:
+    ) -> dict:
         logger.info(f"Creating integration with id: {self.integration_identifier}")
         headers = await self.auth.headers()
         json = {
@@ -71,13 +71,14 @@ class IntegrationClientMixin:
             f"{self.auth.api_url}/integration", headers=headers, json=json
         )
         handle_status_code(response)
+        return response.json()["integration"]
 
     async def patch_integration(
         self,
         _type: str | None = None,
         changelog_destination: dict[str, Any] | None = None,
         port_app_config: Optional["PortAppConfig"] = None,
-    ) -> None:
+    ) -> dict:
         logger.info(f"Updating integration with id: {self.integration_identifier}")
         headers = await self.auth.headers()
         json: dict[str, Any] = {}
@@ -95,34 +96,7 @@ class IntegrationClientMixin:
             json=json,
         )
         handle_status_code(response)
-
-    async def initialize_integration(
-        self,
-        _type: str,
-        changelog_destination: dict[str, Any],
-        port_app_config: Optional["PortAppConfig"] = None,
-    ) -> None:
-        logger.info(f"Initiating integration with id: {self.integration_identifier}")
-        response = await self._get_current_integration()
-        if response.status_code == status.HTTP_404_NOT_FOUND:
-            await self.create_integration(_type, changelog_destination, port_app_config)
-        else:
-            handle_status_code(response)
-
-            integration = response.json()["integration"]
-            logger.info("Checking for diff in integration configuration")
-            if (
-                integration["changelogDestination"] != changelog_destination
-                or integration["installationAppType"] != _type
-                or integration.get("version") != self.integration_version
-            ):
-                await self.patch_integration(
-                    _type, changelog_destination, port_app_config
-                )
-
-        logger.info(
-            f"Integration with id: {self.integration_identifier} successfully registered"
-        )
+        return response.json()["integration"]
 
     async def ingest_integration_logs(self, logs: list[dict[str, Any]]) -> None:
         logger.debug("Ingesting logs")
@@ -135,5 +109,20 @@ class IntegrationClientMixin:
                 "logs": logs,
             },
         )
-        handle_status_code(response)
+        handle_status_code(response, should_log=False)
         logger.debug("Logs successfully ingested")
+
+    async def ingest_integration_kind_examples(
+        self, kind: str, data: list[dict[str, Any]], should_log: bool = True
+    ):
+        logger.debug(f"Ingesting examples for kind: {kind}")
+        headers = await self.auth.headers()
+        response = await self.client.post(
+            f"{self.auth.api_url}/integration/{quote_plus(self.integration_identifier)}/kinds/{quote_plus(kind)}/examples",
+            headers=headers,
+            json={
+                "examples": sensitive_log_filter.mask_object(data, full_hide=True),
+            },
+        )
+        handle_status_code(response, should_log=should_log)
+        logger.debug(f"Examples for kind {kind} successfully ingested")

port_ocean/clients/port/retry_transport.py

@@ -15,11 +15,6 @@ class TokenRetryTransport(RetryTransport):
         super().__init__(**kwargs)
         self.port_client = port_client
 
-    def _is_retryable_method(self, request: httpx.Request) -> bool:
-        return super()._is_retryable_method(request) or request.url.path.endswith(
-            "/auth/access_token"
-        )
-
     async def _handle_unauthorized(self, response: httpx.Response) -> None:
         token = await self.port_client.auth.token
         response.headers["Authorization"] = f"Bearer {token}"

port_ocean/clients/port/utils.py

@@ -16,9 +16,9 @@ if TYPE_CHECKING:
 # period of time, before raising an exception.
 # The max_connections value can't be too high, as it will cause the application to run out of memory.
 # The max_keepalive_connections can't be too high, as it will cause the application to run out of available connections.
-PORT_HTTP_MAX_CONNECTIONS_LIMIT = 200
+PORT_HTTP_MAX_CONNECTIONS_LIMIT = 100
 PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS = 50
-PORT_HTTP_TIMEOUT = 10.0
+PORT_HTTP_TIMEOUT = 60.0
 
 PORT_HTTPX_TIMEOUT = httpx.Timeout(PORT_HTTP_TIMEOUT)
 PORT_HTTPX_LIMITS = httpx.Limits(
@@ -28,13 +28,19 @@ PORT_HTTPX_LIMITS = httpx.Limits(
 
 _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 
+FIVE_MINUETS = 60 * 5
+
 
 def _get_http_client_context(port_client: "PortClient") -> httpx.AsyncClient:
     client = _http_client.top
     if client is None:
         client = OceanAsyncClient(
             TokenRetryTransport,
-            transport_kwargs={"port_client": port_client},
+            transport_kwargs={
+                "port_client": port_client,
+                "max_backoff_wait": FIVE_MINUETS,
+                "base_delay": 0.3,
+            },
             timeout=PORT_HTTPX_TIMEOUT,
             limits=PORT_HTTPX_LIMITS,
         )