port-ocean 0.1.3rc4__tar.gz → 0.1.4.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of port-ocean might be problematic. Click here for more details.

Files changed (104) hide show
  1. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/PKG-INFO +1 -1
  2. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/new.py +6 -1
  3. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +1 -1
  4. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/config/base.py +1 -1
  5. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/event_listener/base.py +7 -0
  6. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/event_listener/factory.py +17 -0
  7. port_ocean-0.1.4.dev1/port_ocean/core/event_listener/http/event_listener.py +66 -0
  8. port_ocean-0.1.4.dev1/port_ocean/core/event_listener/kafka/event_listener.py +141 -0
  9. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/event_listener/polling/event_listener.py +22 -0
  10. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +11 -4
  11. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/integrations/base.py +26 -0
  12. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/integrations/mixins/utils.py +4 -3
  13. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/models.py +2 -14
  14. port_ocean-0.1.4.dev1/port_ocean/exceptions/__init__.py +0 -0
  15. port_ocean-0.1.4.dev1/port_ocean/logger_setup.py +36 -0
  16. port_ocean-0.1.4.dev1/port_ocean/py.typed +0 -0
  17. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/pyproject.toml +1 -1
  18. port_ocean-0.1.3rc4/port_ocean/core/event_listener/http/event_listener.py +0 -43
  19. port_ocean-0.1.3rc4/port_ocean/core/event_listener/kafka/event_listener.py +0 -94
  20. port_ocean-0.1.3rc4/port_ocean/logger_setup.py +0 -22
  21. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/LICENSE +0 -0
  22. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/README.md +0 -0
  23. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/__init__.py +0 -0
  24. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/__init__.py +0 -0
  25. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cli.py +0 -0
  26. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/__init__.py +0 -0
  27. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/list_integrations.py +0 -0
  28. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/main.py +0 -0
  29. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/pull.py +0 -0
  30. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/sail.py +0 -0
  31. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/commands/version.py +0 -0
  32. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
  33. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.dockerignore +0 -0
  34. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
  35. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
  36. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
  37. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
  38. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile +0 -0
  39. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile +0 -0
  40. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
  41. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
  42. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/config.yaml +0 -0
  43. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
  44. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
  45. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
  46. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py +0 -0
  47. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/cli/utils.py +0 -0
  48. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/__init__.py +0 -0
  49. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/__init__.py +0 -0
  50. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/authentication.py +0 -0
  51. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/client.py +0 -0
  52. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/mixins/__init__.py +0 -0
  53. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/mixins/blueprints.py +0 -0
  54. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/mixins/entities.py +0 -0
  55. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/mixins/integrations.py +0 -0
  56. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/types.py +0 -0
  57. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/clients/port/utils.py +0 -0
  58. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/config/__init__.py +0 -0
  59. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/config/dynamic.py +0 -0
  60. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/config/settings.py +0 -0
  61. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/consumers/__init__.py +0 -0
  62. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/consumers/base_consumer.py +0 -0
  63. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/consumers/kafka_consumer.py +0 -0
  64. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/context/__init__.py +0 -0
  65. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/context/event.py +0 -0
  66. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/context/ocean.py +0 -0
  67. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/__init__.py +0 -0
  68. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/base.py +0 -0
  69. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/event_listener/__init__.py +0 -0
  70. {port_ocean-0.1.3rc4/port_ocean/core/handlers/entities_state_applier → port_ocean-0.1.4.dev1/port_ocean/core/event_listener/http}/__init__.py +0 -0
  71. {port_ocean-0.1.3rc4/port_ocean/core/handlers/entities_state_applier/port → port_ocean-0.1.4.dev1/port_ocean/core/event_listener/kafka}/__init__.py +0 -0
  72. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/event_listener/polling/utils.py +0 -0
  73. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/__init__.py +0 -0
  74. {port_ocean-0.1.3rc4/port_ocean/core/handlers/entity_processor → port_ocean-0.1.4.dev1/port_ocean/core/handlers/entities_state_applier}/__init__.py +0 -0
  75. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
  76. {port_ocean-0.1.3rc4/port_ocean/core/handlers/port_app_config → port_ocean-0.1.4.dev1/port_ocean/core/handlers/entities_state_applier/port}/__init__.py +0 -0
  77. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
  78. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
  79. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py +0 -0
  80. {port_ocean-0.1.3rc4/port_ocean/core/integrations → port_ocean-0.1.4.dev1/port_ocean/core/handlers/entity_processor}/__init__.py +0 -0
  81. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entity_processor/base.py +0 -0
  82. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +0 -0
  83. {port_ocean-0.1.3rc4/port_ocean/core/integrations/mixins → port_ocean-0.1.4.dev1/port_ocean/core/handlers/port_app_config}/__init__.py +0 -0
  84. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/port_app_config/api.py +0 -0
  85. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/port_app_config/base.py +0 -0
  86. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/handlers/port_app_config/models.py +0 -0
  87. {port_ocean-0.1.3rc4/port_ocean/exceptions → port_ocean-0.1.4.dev1/port_ocean/core/integrations}/__init__.py +0 -0
  88. /port_ocean-0.1.3rc4/port_ocean/py.typed → /port_ocean-0.1.4.dev1/port_ocean/core/integrations/mixins/__init__.py +0 -0
  89. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/integrations/mixins/events.py +0 -0
  90. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/integrations/mixins/handler.py +0 -0
  91. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/integrations/mixins/sync.py +0 -0
  92. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/ocean_types.py +0 -0
  93. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/core/utils.py +0 -0
  94. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/api.py +0 -0
  95. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/base.py +0 -0
  96. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/clients.py +0 -0
  97. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/context.py +0 -0
  98. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/core.py +0 -0
  99. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/exceptions/port_defaults.py +0 -0
  100. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/middlewares.py +0 -0
  101. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/ocean.py +0 -0
  102. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/port_defaults.py +0 -0
  103. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/run.py +0 -0
  104. {port_ocean-0.1.3rc4 → port_ocean-0.1.4.dev1}/port_ocean/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: port-ocean
3
- Version: 0.1.3rc4
3
+ Version: 0.1.4.dev1
4
4
  Summary: Port Ocean is a CLI tool for managing your Port projects.
5
5
  Home-page: https://app.getport.io
6
6
  Keywords: ocean,port-ocean,port
@@ -3,6 +3,7 @@
3
3
  import click
4
4
  from cookiecutter.main import cookiecutter # type: ignore
5
5
 
6
+ from port_ocean import __version__
6
7
  from port_ocean.cli.commands.main import cli_start, print_logo, console
7
8
  from port_ocean.cli.utils import cli_root_path
8
9
 
@@ -21,7 +22,11 @@ def new(path: str) -> None:
21
22
  "🚢 Unloading cargo... Setting up your integration at the dock.", style="bold"
22
23
  )
23
24
 
24
- result = cookiecutter(f"{cli_root_path}/cookiecutter", output_dir=path)
25
+ result = cookiecutter(
26
+ f"{cli_root_path}/cookiecutter",
27
+ output_dir=path,
28
+ extra_context={"version": __version__},
29
+ )
25
30
  name = result.split("/")[-1]
26
31
 
27
32
  console.print(
@@ -6,7 +6,7 @@ authors = ["{{cookiecutter.full_name}} <{{cookiecutter.email}}>"]
6
6
 
7
7
  [tool.poetry.dependencies]
8
8
  python = "^3.11"
9
- port_ocean = { version = "^0.1.0", extras = ["cli"] }
9
+ port_ocean = { version = "^{{ cookiecutter.version }}", extras = ["cli"] }
10
10
 
11
11
  [tool.poetry.group.dev.dependencies]
12
12
  pytest = "^7.2"
@@ -8,7 +8,7 @@ from humps import decamelize
8
8
  from pydantic import BaseSettings
9
9
  from pydantic.env_settings import EnvSettingsSource, InitSettingsSource
10
10
 
11
- PROVIDER_WRAPPER_PATTERN = r"\\{\\{ from (.*) \\}\\}"
11
+ PROVIDER_WRAPPER_PATTERN = r"{{ from (.*) }}"
12
12
  PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$"
13
13
 
14
14
 
@@ -5,6 +5,9 @@ from pydantic import BaseModel, Extra
5
5
 
6
6
 
7
7
  class EventListenerEvents(TypedDict):
8
+ """
9
+ A dictionary containing event types and their corresponding event handlers.
10
+ """
8
11
  on_resync: Callable[[dict[Any, Any]], Awaitable[None]]
9
12
 
10
13
 
@@ -24,4 +27,8 @@ class EventListenerSettings(BaseModel, extra=Extra.allow):
24
27
  type: str
25
28
 
26
29
  def to_request(self) -> dict[str, Any]:
30
+ """
31
+ Converts the Settings object to a dictionary representation (request format).
32
+ This method is used when configuring the event listener settings for the Polling event listener.
33
+ """
27
34
  return {"type": self.type}
@@ -20,6 +20,16 @@ from port_ocean.exceptions.core import UnsupportedEventListenerTypeException
20
20
 
21
21
 
22
22
  class EventListenerFactory(BaseWithContext):
23
+ """
24
+ This class is responsible for creating different types of event listeners based on the provided configuration.
25
+ The factory takes a PortOceanContext, installation_id, and events dictionary as parameters.
26
+
27
+ Parameters:
28
+ context (PortOceanContext): The PortOceanContext object containing information about the current application context.
29
+ installation_id (str): The identifier of the installation associated with the event listener.
30
+ events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
31
+
32
+ """
23
33
  def __init__(
24
34
  self,
25
35
  context: PortOceanContext,
@@ -31,6 +41,13 @@ class EventListenerFactory(BaseWithContext):
31
41
  self.events = events
32
42
 
33
43
  async def create_event_listener(self) -> BaseEventListener:
44
+ """
45
+ Creates and returns a specific event listener based on the provided configuration.
46
+ The event listener is wrapped with the events provided in the factory's initialization.
47
+
48
+ Raises:
49
+ UnsupportedEventListenerTypeException: If the event listener type is not supported.
50
+ """
34
51
  wrapped_events: EventListenerEvents = {"on_resync": self.events["on_resync"]}
35
52
  event_listener: BaseEventListener
36
53
  config = self.context.config.event_listener
@@ -0,0 +1,66 @@
1
+ from typing import Literal, Any
2
+
3
+ from fastapi import APIRouter
4
+ from loguru import logger
5
+ from pydantic import AnyHttpUrl
6
+
7
+ from port_ocean.context.ocean import ocean
8
+ from port_ocean.core.event_listener.base import (
9
+ BaseEventListener,
10
+ EventListenerEvents,
11
+ EventListenerSettings,
12
+ )
13
+
14
+
15
+ class HttpEventListenerSettings(EventListenerSettings):
16
+ """
17
+ This class inherits from `EventListenerSettings`, which provides a foundation for defining event listener configurations.
18
+ The `HttpEventListenerSettings` specifically includes settings related to the HTTP event listener (Webhook).
19
+
20
+ Attributes:
21
+ type (Literal["WEBHOOK"]): A literal indicating the type of the event listener, which is set to "WEBHOOK" for this class.
22
+ app_host (AnyHttpUrl): The base URL of the application hosting the webhook.
23
+ The "AnyHttpUrl" type indicates that the value must be a valid HTTP/HTTPS URL.
24
+ """
25
+ type: Literal["WEBHOOK"]
26
+ app_host: AnyHttpUrl
27
+
28
+ def to_request(self) -> dict[str, Any]:
29
+ return {
30
+ **super().to_request(),
31
+ "url": self.app_host + "/resync",
32
+ }
33
+
34
+
35
+ class HttpEventListener(BaseEventListener):
36
+ """
37
+ HTTP event listener that listens for webhook events and triggers "on_resync" event.
38
+
39
+ This class inherits from `BaseEventListener`, which provides a foundation for creating event listeners.
40
+ The `HttpEventListener` listens for HTTP POST requests to the `/resync` endpoint and triggers the "on_resync" event.
41
+
42
+ Parameters:
43
+ events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
44
+ event_listener_config (HttpEventListenerSettings): Configuration settings for the HTTP event listener.
45
+ """
46
+ def __init__(
47
+ self,
48
+ events: EventListenerEvents,
49
+ event_listener_config: HttpEventListenerSettings,
50
+ ):
51
+ super().__init__(events)
52
+ self.event_listener_config = event_listener_config
53
+
54
+ async def start(self) -> None:
55
+ """
56
+ Starts the HTTP event listener.
57
+ It sets up an APIRouter to handle the `/resync` endpoint and registers the "on_resync" event handler.
58
+ """
59
+ logger.info("Setting up HTTP Event Listener")
60
+ target_channel_router = APIRouter()
61
+
62
+ @target_channel_router.post("/resync")
63
+ async def resync() -> None:
64
+ await self.events["on_resync"]({})
65
+
66
+ ocean.app.fast_api_app.include_router(target_channel_router)
@@ -0,0 +1,141 @@
1
+ import threading
2
+ from typing import Any, Callable, Literal
3
+
4
+ from loguru import logger
5
+
6
+ from port_ocean.consumers.kafka_consumer import KafkaConsumer, KafkaConsumerConfig
7
+ from port_ocean.context.ocean import (
8
+ PortOceanContext,
9
+ initialize_port_ocean_context,
10
+ ocean,
11
+ )
12
+ from port_ocean.core.event_listener.base import (
13
+ BaseEventListener,
14
+ EventListenerEvents,
15
+ EventListenerSettings,
16
+ )
17
+
18
+
19
+ class KafkaEventListenerSettings(EventListenerSettings):
20
+ """
21
+ This class inherits from `EventListenerSettings`, which provides a foundation for defining event listener configurations.
22
+ The `KafkaEventListenerSettings` specifically includes settings related to the Kafka event listener.
23
+
24
+ Attributes:
25
+ type (Literal["KAFKA"]): A literal indicating the type of the event listener, which is set to "KAFKA" for this class.
26
+ brokers (str): The comma-separated list of Kafka broker URLs to connect to.
27
+ security_protocol (str): The security protocol used for communication with Kafka brokers.
28
+ The default value is "SASL_SSL".
29
+ authentication_mechanism (str): The authentication mechanism used for secure access to Kafka brokers.
30
+ The default value is "SCRAM-SHA-512".
31
+ kafka_security_enabled (bool): A flag indicating whether Kafka security is enabled.
32
+ If True, credentials and security settings are used to connect to Kafka.
33
+ The default value is True.
34
+ consumer_poll_timeout (int): The maximum time in seconds to wait for messages during a poll.
35
+ The default value is 1 second.
36
+ """
37
+ type: Literal["KAFKA"]
38
+ brokers: str = "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"
39
+ security_protocol: str = "SASL_SSL"
40
+ authentication_mechanism: str = "SCRAM-SHA-512"
41
+ kafka_security_enabled: bool = True
42
+ consumer_poll_timeout: int = 1
43
+
44
+
45
+ class KafkaEventListener(BaseEventListener):
46
+ """
47
+ The `KafkaEventListener` specifically listens for messages from a Kafka consumer related to changes in an integration.
48
+
49
+ Parameters:
50
+ events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
51
+ event_listener_config (KafkaEventListenerSettings): Configuration settings for the Kafka event listener.
52
+ org_id (str): The identifier of the organization associated with the integration.
53
+ integration_identifier (str): The identifier of the integration being monitored.
54
+ integration_type (str): The type of the integration being monitored.
55
+ """
56
+ def __init__(
57
+ self,
58
+ events: EventListenerEvents,
59
+ event_listener_config: KafkaEventListenerSettings,
60
+ org_id: str,
61
+ integration_identifier: str,
62
+ integration_type: str,
63
+ ):
64
+ super().__init__(events)
65
+ self.event_listener_config = event_listener_config
66
+ self.org_id = org_id
67
+ self.integration_identifier = integration_identifier
68
+ self.integration_type = integration_type
69
+
70
+ async def _get_kafka_config(self) -> KafkaConsumerConfig:
71
+ """
72
+ A private method that returns the Kafka consumer configuration based on the provided settings.
73
+ If Kafka security is enabled, it fetches Kafka credentials using the ocean.port_client.get_kafka_creds() method.
74
+ Otherwise, it returns the KafkaConsumerConfig object parsed from the event_listener_config.
75
+ """
76
+ if self.event_listener_config.kafka_security_enabled:
77
+ creds = await ocean.port_client.get_kafka_creds()
78
+ return KafkaConsumerConfig(
79
+ **self.event_listener_config.dict(),
80
+ username=creds.get("username"),
81
+ password=creds.get("password"),
82
+ group_name=f"{self.integration_type}.{self.integration_identifier}",
83
+ )
84
+
85
+ return KafkaConsumerConfig.parse_obj(self.event_listener_config.dict())
86
+
87
+ def _should_be_processed(self, msg_value: dict[Any, Any], topic: str) -> bool:
88
+ """
89
+ Determines if a given message should be processed based on the integration identifier and topic.
90
+ Returns True if the message should be processed, False otherwise.
91
+ """
92
+ integration_identifier = (
93
+ msg_value.get("diff", {}).get("after", {}).get("identifier")
94
+ )
95
+ if integration_identifier == self.integration_identifier and (
96
+ "change.log" in topic
97
+ ):
98
+ return msg_value.get("changelogDestination", {}).get("type", "") == "KAFKA"
99
+
100
+ return False
101
+
102
+ async def _handle_message(self, message: dict[Any, Any], topic: str) -> None:
103
+ """
104
+ A private method that handles incoming Kafka messages.
105
+ If the message should be processed (determined by `_should_be_processed`), it triggers the corresponding event handler.
106
+ """
107
+ if not self._should_be_processed(message, topic):
108
+ return
109
+
110
+ if "change.log" in topic and message is not None:
111
+ await self.events["on_resync"](message)
112
+
113
+ def _wrapped_start(
114
+ self, context: PortOceanContext, func: Callable[[], None]
115
+ ) -> Callable[[], None]:
116
+ """
117
+ A method that wraps the `start` method, initializing the PortOceanContext and invoking the given function.
118
+ """
119
+ ocean_app = context.app
120
+
121
+ def wrapper() -> None:
122
+ initialize_port_ocean_context(ocean_app=ocean_app)
123
+ func()
124
+
125
+ return wrapper
126
+
127
+ async def start(self) -> None:
128
+ """
129
+ The main method that starts the Kafka consumer.
130
+ It creates a KafkaConsumer instance with the given configuration and starts it in a separate thread.
131
+ """
132
+ consumer = KafkaConsumer(
133
+ msg_process=self._handle_message,
134
+ config=await self._get_kafka_config(),
135
+ org_id=self.org_id,
136
+ )
137
+ logger.info("Starting Kafka consumer")
138
+ threading.Thread(
139
+ name="ocean_kafka_consumer",
140
+ target=self._wrapped_start(ocean, consumer.start),
141
+ ).start()
@@ -12,6 +12,14 @@ from port_ocean.core.event_listener.polling.utils import repeat_every
12
12
 
13
13
 
14
14
  class PollingEventListenerSettings(EventListenerSettings):
15
+ """
16
+ Attributes:
17
+ type (Literal["POLLING"]): A literal indicating the type of the event listener, which is set to "POLLING" for this class.
18
+ resync_on_start (bool): A flag indicating whether to trigger a resync event on the start of the polling event listener.
19
+ If True, the "on_resync" event will be triggered immediately when the polling listener starts.
20
+ interval (int): The interval in seconds at which the polling event listener checks for changes in the integration.
21
+ The default interval is set to 60 seconds.
22
+ """
15
23
  type: Literal["POLLING"]
16
24
  resync_on_start: bool = True
17
25
  interval: int = 60
@@ -21,6 +29,15 @@ class PollingEventListenerSettings(EventListenerSettings):
21
29
 
22
30
 
23
31
  class PollingEventListener(BaseEventListener):
32
+ """
33
+ Polling event listener that checks for changes in the integration every `interval` seconds.
34
+
35
+ The `PollingEventListener` periodically checks for changes in the integration and triggers the "on_resync" event if changes are detected.
36
+
37
+ Parameters:
38
+ events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
39
+ event_listener_config (PollingEventListenerSettings): Configuration settings for the Polling event listener.
40
+ """
24
41
  def __init__(
25
42
  self,
26
43
  events: EventListenerEvents,
@@ -31,6 +48,11 @@ class PollingEventListener(BaseEventListener):
31
48
  self._last_updated_at = None
32
49
 
33
50
  async def start(self) -> None:
51
+ """
52
+ Starts the polling event listener.
53
+ It registers the "on_resync" event to be called every `interval` seconds specified in the `event_listener_config`.
54
+ The `on_resync` event is triggered if the integration has changed since the last update.
55
+ """
34
56
  logger.info(
35
57
  f"Setting up Polling event listener with interval: {self.event_listener_config.interval}"
36
58
  )
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ from collections import defaultdict
2
3
  from itertools import groupby
3
4
 
4
5
  from port_ocean.clients.port.client import PortClient
@@ -33,12 +34,18 @@ async def get_related_entities(
33
34
  for entity in entities_with_relations
34
35
  ]
35
36
 
36
- related_entities = []
37
+ blueprints_to_relations = defaultdict(list)
37
38
  for entity, blueprint in entity_to_blueprint:
38
39
  for relation_name, relation in entity.relations.items():
39
40
  relation_blueprint = blueprint.relations[relation_name].target
40
- related_entities.append(
41
- Entity(identifier=relation, blueprint=relation_blueprint)
41
+ blueprints_to_relations[relation_blueprint].extend(
42
+ relation if isinstance(relation, list) else [relation]
42
43
  )
43
44
 
44
- return related_entities
45
+ return [
46
+ Entity(identifier=relation, blueprint=blueprint)
47
+ for blueprint, relations in blueprints_to_relations.items()
48
+ # multiple entities can point to the same relation in the same blueprint, for performance reasons
49
+ # we want to avoid fetching the same relation multiple times
50
+ for relation in set(relations)
51
+ ]
@@ -15,6 +15,29 @@ from port_ocean.exceptions.core import IntegrationAlreadyStartedException
15
15
 
16
16
 
17
17
  class BaseIntegration(SyncRawMixin, SyncMixin):
18
+ """
19
+ This is the default integration class that Ocean initializes when no custom integration class is
20
+ provided.
21
+
22
+ This class provides a foundation for implementing various integration types. It inherits from
23
+ both SyncRawMixin and SyncMixin, which provide synchronization and event handling functionality.
24
+
25
+ Parameters:
26
+ context (PortOceanContext): The PortOceanContext object providing the necessary context
27
+ for the integration.
28
+
29
+ Attributes:
30
+ started (bool): Flag indicating whether the integration has been started.
31
+ context (PortOceanContext): The PortOceanContext object containing integration context.
32
+ event_listener_factory (EventListenerFactory): Factory to create event listeners for
33
+ handling integration events.
34
+
35
+ Raises:
36
+ IntegrationAlreadyStartedException: Raised if the integration is attempted to be started
37
+ more than once.
38
+ NotImplementedError: Raised if the `on_resync` method is not implemented, and the event
39
+ strategy does not have a custom implementation for resync events.
40
+ """
18
41
  def __init__(self, context: PortOceanContext):
19
42
  SyncRawMixin.__init__(self)
20
43
  SyncMixin.__init__(self)
@@ -27,6 +50,9 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
27
50
  )
28
51
 
29
52
  async def start(self) -> None:
53
+ """
54
+ Initializes handlers, establishes integration at the specified port, and starts the event listener.
55
+ """
30
56
  logger.info("Starting integration")
31
57
  if self.started:
32
58
  raise IntegrationAlreadyStartedException("Integration already started")
@@ -1,6 +1,7 @@
1
1
  from contextlib import contextmanager
2
2
  from typing import Awaitable, Generator, Callable
3
3
 
4
+ from loguru import logger
4
5
  from port_ocean.core.ocean_types import (
5
6
  ASYNC_GENERATOR_RESYNC_TYPE,
6
7
  RAW_RESULT,
@@ -20,9 +21,9 @@ def resync_error_handling() -> Generator[None, None, None]:
20
21
  except StopAsyncIteration:
21
22
  raise
22
23
  except Exception as error:
23
- raise OceanAbortException(
24
- f"Failed to execute resync function, error: {error}"
25
- ) from error
24
+ err_msg = f"Failed to execute resync function, error: {error}"
25
+ logger.exception(err_msg)
26
+ raise OceanAbortException(err_msg) from error
26
27
 
27
28
 
28
29
  async def resync_function_wrapper(
@@ -10,7 +10,7 @@ class Entity(BaseModel):
10
10
  identifier: str
11
11
  blueprint: str
12
12
  title: str | None
13
- team: str | list[str] = []
13
+ team: str | None | list[str | None] = []
14
14
  properties: dict[str, Any] = {}
15
15
  relations: dict[str, Any] = {}
16
16
 
@@ -18,19 +18,7 @@ class Entity(BaseModel):
18
18
  def parse_obj(cls: Type["Model"], obj: dict[Any, Any]) -> "Model":
19
19
  obj["identifier"] = str(obj.get("identifier"))
20
20
  obj["blueprint"] = str(obj.get("blueprint"))
21
- if obj.get("team"):
22
- team = obj.get("team")
23
- obj["team"] = (
24
- [str(item) for item in team]
25
- if isinstance(team, list)
26
- else str(obj.get("team"))
27
- )
28
-
29
- for key, value in obj.get("relations", {}).items():
30
- if isinstance(value, list):
31
- obj["relations"][key] = [str(item) for item in value]
32
- else:
33
- obj["relations"][key] = str(value)
21
+
34
22
  return super(Entity, cls).parse_obj(obj)
35
23
 
36
24
 
@@ -0,0 +1,36 @@
1
+ import sys
2
+
3
+ import loguru
4
+ from loguru import logger
5
+
6
+ from port_ocean.config.settings import LogLevelType
7
+
8
+
9
+ def setup_logger(level: LogLevelType) -> None:
10
+ logger_format = (
11
+ "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
12
+ "<level>{level: <8}</level> | "
13
+ "<level>{message}</level> | {extra}"
14
+ )
15
+
16
+ logger.remove()
17
+ logger.add(
18
+ sys.stdout,
19
+ level=level.upper(),
20
+ format=logger_format,
21
+ enqueue=True, # process logs in background
22
+ diagnose=False, # hide variable values in log backtrace
23
+ )
24
+ logger.configure(patcher=exception_deserializer)
25
+
26
+
27
+ def exception_deserializer(record: "loguru.Record") -> None:
28
+ """
29
+ Workaround for when trying to log exception objects with loguru.
30
+ loguru doesn't able to deserialize `Exception` subclasses.
31
+ https://github.com/Delgan/loguru/issues/504#issuecomment-917365972
32
+ """
33
+ exception: loguru.RecordException | None = record["exception"]
34
+ if exception is not None:
35
+ fixed = Exception(str(exception.value))
36
+ record["exception"] = exception._replace(value=fixed)
File without changes
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "port-ocean"
3
- version = "0.1.3rc4"
3
+ version = "0.1.4dev1"
4
4
  description = "Port Ocean is a CLI tool for managing your Port projects."
5
5
  readme = "README.md"
6
6
  homepage = "https://app.getport.io"
@@ -1,43 +0,0 @@
1
- from typing import Literal, Any
2
-
3
- from fastapi import APIRouter
4
- from loguru import logger
5
- from pydantic import AnyHttpUrl
6
-
7
- from port_ocean.context.ocean import ocean
8
- from port_ocean.core.event_listener.base import (
9
- BaseEventListener,
10
- EventListenerEvents,
11
- EventListenerSettings,
12
- )
13
-
14
-
15
- class HttpEventListenerSettings(EventListenerSettings):
16
- type: Literal["WEBHOOK"]
17
- app_host: AnyHttpUrl
18
-
19
- def to_request(self) -> dict[str, Any]:
20
- return {
21
- **super().to_request(),
22
- "url": self.app_host + "/resync",
23
- }
24
-
25
-
26
- class HttpEventListener(BaseEventListener):
27
- def __init__(
28
- self,
29
- events: EventListenerEvents,
30
- event_listener_config: HttpEventListenerSettings,
31
- ):
32
- super().__init__(events)
33
- self.event_listener_config = event_listener_config
34
-
35
- async def start(self) -> None:
36
- logger.info("Setting up HTTP Event Listener")
37
- target_channel_router = APIRouter()
38
-
39
- @target_channel_router.post("/resync")
40
- async def resync() -> None:
41
- await self.events["on_resync"]({})
42
-
43
- ocean.app.fast_api_app.include_router(target_channel_router)
@@ -1,94 +0,0 @@
1
- import threading
2
- from typing import Any, Callable, Literal
3
-
4
- from loguru import logger
5
-
6
- from port_ocean.consumers.kafka_consumer import KafkaConsumer, KafkaConsumerConfig
7
- from port_ocean.context.ocean import (
8
- PortOceanContext,
9
- initialize_port_ocean_context,
10
- ocean,
11
- )
12
- from port_ocean.core.event_listener.base import (
13
- BaseEventListener,
14
- EventListenerEvents,
15
- EventListenerSettings,
16
- )
17
-
18
-
19
- class KafkaEventListenerSettings(EventListenerSettings):
20
- type: Literal["KAFKA"]
21
- brokers: str = "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"
22
- security_protocol: str = "SASL_SSL"
23
- authentication_mechanism: str = "SCRAM-SHA-512"
24
- kafka_security_enabled: bool = True
25
- consumer_poll_timeout: int = 1
26
-
27
-
28
- class KafkaEventListener(BaseEventListener):
29
- def __init__(
30
- self,
31
- events: EventListenerEvents,
32
- event_listener_config: KafkaEventListenerSettings,
33
- org_id: str,
34
- integration_identifier: str,
35
- integration_type: str,
36
- ):
37
- super().__init__(events)
38
- self.event_listener_config = event_listener_config
39
- self.org_id = org_id
40
- self.integration_identifier = integration_identifier
41
- self.integration_type = integration_type
42
-
43
- async def _get_kafka_config(self) -> KafkaConsumerConfig:
44
- if self.event_listener_config.kafka_security_enabled:
45
- creds = await ocean.port_client.get_kafka_creds()
46
- return KafkaConsumerConfig(
47
- **self.event_listener_config.dict(),
48
- username=creds.get("username"),
49
- password=creds.get("password"),
50
- group_name=f"{self.integration_type}.{self.integration_identifier}",
51
- )
52
-
53
- return KafkaConsumerConfig.parse_obj(self.event_listener_config.dict())
54
-
55
- def should_be_processed(self, msg_value: dict[Any, Any], topic: str) -> bool:
56
- integration_identifier = (
57
- msg_value.get("diff", {}).get("after", {}).get("identifier")
58
- )
59
- if integration_identifier == self.integration_identifier and (
60
- "change.log" in topic
61
- ):
62
- return msg_value.get("changelogDestination", {}).get("type", "") == "KAFKA"
63
-
64
- return False
65
-
66
- async def _handle_message(self, message: dict[Any, Any], topic: str) -> None:
67
- if not self.should_be_processed(message, topic):
68
- return
69
-
70
- if "change.log" in topic and message is not None:
71
- await self.events["on_resync"](message)
72
-
73
- def wrapped_start(
74
- self, context: PortOceanContext, func: Callable[[], None]
75
- ) -> Callable[[], None]:
76
- ocean_app = context.app
77
-
78
- def wrapper() -> None:
79
- initialize_port_ocean_context(ocean_app=ocean_app)
80
- func()
81
-
82
- return wrapper
83
-
84
- async def start(self) -> None:
85
- consumer = KafkaConsumer(
86
- msg_process=self._handle_message,
87
- config=await self._get_kafka_config(),
88
- org_id=self.org_id,
89
- )
90
- logger.info("Starting Kafka consumer")
91
- threading.Thread(
92
- name="ocean_kafka_consumer",
93
- target=self.wrapped_start(ocean, consumer.start),
94
- ).start()
@@ -1,22 +0,0 @@
1
- import sys
2
-
3
- from loguru import logger
4
-
5
- from port_ocean.config.settings import LogLevelType
6
-
7
-
8
- def setup_logger(level: LogLevelType) -> None:
9
- logger_format = (
10
- "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
11
- "<level>{level: <8}</level> | "
12
- "<level>{message}</level> | {extra}"
13
- )
14
-
15
- logger.remove()
16
- logger.add(
17
- sys.stdout,
18
- level=level.upper(),
19
- format=logger_format,
20
- enqueue=True, # process logs in background
21
- diagnose=False, # hide variable values in log backtrace
22
- )
File without changes
File without changes