port-ocean 0.5.5__py3-none-any.whl → 0.17.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of port-ocean might be problematic. Click here for more details.

Files changed (112) hide show
  1. integrations/_infra/Dockerfile.Deb +56 -0
  2. integrations/_infra/Dockerfile.alpine +108 -0
  3. integrations/_infra/Dockerfile.base.builder +26 -0
  4. integrations/_infra/Dockerfile.base.runner +13 -0
  5. integrations/_infra/Dockerfile.dockerignore +94 -0
  6. {port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}} → integrations/_infra}/Makefile +21 -8
  7. integrations/_infra/grpcio.sh +18 -0
  8. integrations/_infra/init.sh +5 -0
  9. port_ocean/bootstrap.py +1 -1
  10. port_ocean/cli/commands/defaults/clean.py +3 -1
  11. port_ocean/cli/commands/new.py +42 -7
  12. port_ocean/cli/commands/sail.py +7 -1
  13. port_ocean/cli/cookiecutter/cookiecutter.json +3 -0
  14. port_ocean/cli/cookiecutter/hooks/post_gen_project.py +20 -3
  15. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +6 -0
  16. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +41 -0
  17. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +16 -0
  18. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +6 -7
  19. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +1 -1
  20. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +7 -0
  21. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +1 -0
  22. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +16 -1
  23. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +21 -10
  24. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +2 -0
  25. port_ocean/clients/port/authentication.py +16 -4
  26. port_ocean/clients/port/client.py +17 -0
  27. port_ocean/clients/port/mixins/blueprints.py +7 -8
  28. port_ocean/clients/port/mixins/entities.py +108 -53
  29. port_ocean/clients/port/mixins/integrations.py +23 -34
  30. port_ocean/clients/port/retry_transport.py +0 -5
  31. port_ocean/clients/port/utils.py +9 -3
  32. port_ocean/config/base.py +16 -16
  33. port_ocean/config/dynamic.py +2 -0
  34. port_ocean/config/settings.py +79 -11
  35. port_ocean/context/event.py +18 -5
  36. port_ocean/context/ocean.py +14 -3
  37. port_ocean/core/defaults/clean.py +10 -3
  38. port_ocean/core/defaults/common.py +25 -9
  39. port_ocean/core/defaults/initialize.py +111 -100
  40. port_ocean/core/event_listener/__init__.py +8 -0
  41. port_ocean/core/event_listener/base.py +49 -10
  42. port_ocean/core/event_listener/factory.py +9 -1
  43. port_ocean/core/event_listener/http.py +11 -3
  44. port_ocean/core/event_listener/kafka.py +24 -5
  45. port_ocean/core/event_listener/once.py +96 -4
  46. port_ocean/core/event_listener/polling.py +16 -14
  47. port_ocean/core/event_listener/webhooks_only.py +41 -0
  48. port_ocean/core/handlers/__init__.py +1 -2
  49. port_ocean/core/handlers/entities_state_applier/base.py +4 -1
  50. port_ocean/core/handlers/entities_state_applier/port/applier.py +29 -87
  51. port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +5 -2
  52. port_ocean/core/handlers/entity_processor/base.py +26 -22
  53. port_ocean/core/handlers/entity_processor/jq_entity_processor.py +253 -45
  54. port_ocean/core/handlers/port_app_config/base.py +55 -15
  55. port_ocean/core/handlers/port_app_config/models.py +24 -5
  56. port_ocean/core/handlers/resync_state_updater/__init__.py +5 -0
  57. port_ocean/core/handlers/resync_state_updater/updater.py +84 -0
  58. port_ocean/core/integrations/base.py +5 -7
  59. port_ocean/core/integrations/mixins/events.py +3 -1
  60. port_ocean/core/integrations/mixins/sync.py +4 -2
  61. port_ocean/core/integrations/mixins/sync_raw.py +209 -74
  62. port_ocean/core/integrations/mixins/utils.py +1 -1
  63. port_ocean/core/models.py +44 -0
  64. port_ocean/core/ocean_types.py +29 -11
  65. port_ocean/core/utils/entity_topological_sorter.py +90 -0
  66. port_ocean/core/utils/utils.py +109 -0
  67. port_ocean/debug_cli.py +5 -0
  68. port_ocean/exceptions/core.py +4 -0
  69. port_ocean/exceptions/port_defaults.py +0 -2
  70. port_ocean/helpers/retry.py +85 -24
  71. port_ocean/log/handlers.py +23 -2
  72. port_ocean/log/logger_setup.py +8 -1
  73. port_ocean/log/sensetive.py +25 -10
  74. port_ocean/middlewares.py +10 -2
  75. port_ocean/ocean.py +57 -24
  76. port_ocean/run.py +10 -5
  77. port_ocean/tests/__init__.py +0 -0
  78. port_ocean/tests/clients/port/mixins/test_entities.py +53 -0
  79. port_ocean/tests/conftest.py +4 -0
  80. port_ocean/tests/core/defaults/test_common.py +166 -0
  81. port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +350 -0
  82. port_ocean/tests/core/handlers/mixins/test_sync_raw.py +552 -0
  83. port_ocean/tests/core/test_utils.py +73 -0
  84. port_ocean/tests/core/utils/test_entity_topological_sorter.py +99 -0
  85. port_ocean/tests/helpers/__init__.py +0 -0
  86. port_ocean/tests/helpers/fake_port_api.py +191 -0
  87. port_ocean/tests/helpers/fixtures.py +46 -0
  88. port_ocean/tests/helpers/integration.py +31 -0
  89. port_ocean/tests/helpers/ocean_app.py +66 -0
  90. port_ocean/tests/helpers/port_client.py +21 -0
  91. port_ocean/tests/helpers/smoke_test.py +82 -0
  92. port_ocean/tests/log/test_handlers.py +71 -0
  93. port_ocean/tests/test_smoke.py +74 -0
  94. port_ocean/tests/utils/test_async_iterators.py +45 -0
  95. port_ocean/tests/utils/test_cache.py +189 -0
  96. port_ocean/utils/async_iterators.py +109 -0
  97. port_ocean/utils/cache.py +37 -1
  98. port_ocean/utils/misc.py +22 -4
  99. port_ocean/utils/queue_utils.py +88 -0
  100. port_ocean/utils/signal.py +1 -4
  101. port_ocean/utils/time.py +54 -0
  102. {port_ocean-0.5.5.dist-info → port_ocean-0.17.8.dist-info}/METADATA +27 -19
  103. port_ocean-0.17.8.dist-info/RECORD +164 -0
  104. {port_ocean-0.5.5.dist-info → port_ocean-0.17.8.dist-info}/WHEEL +1 -1
  105. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.dockerignore +0 -94
  106. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile +0 -15
  107. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/config.yaml +0 -17
  108. port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py +0 -40
  109. port_ocean/core/utils.py +0 -65
  110. port_ocean-0.5.5.dist-info/RECORD +0 -129
  111. {port_ocean-0.5.5.dist-info → port_ocean-0.17.8.dist-info}/LICENSE.md +0 -0
  112. {port_ocean-0.5.5.dist-info → port_ocean-0.17.8.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,109 @@
1
+ import asyncio
2
+ from typing import Iterable, Any, TypeVar, Callable, Awaitable
3
+
4
+ from loguru import logger
5
+ from pydantic import parse_obj_as, ValidationError
6
+
7
+ from port_ocean.clients.port.client import PortClient
8
+ from port_ocean.core.models import Entity, Runtime
9
+ from port_ocean.core.models import EntityPortDiff
10
+ from port_ocean.core.ocean_types import RAW_RESULT
11
+ from port_ocean.exceptions.core import (
12
+ RawObjectValidationException,
13
+ IntegrationRuntimeException,
14
+ )
15
+
16
+ T = TypeVar("T", bound=tuple[list[Any], ...])
17
+
18
+
19
+ def zip_and_sum(collection: Iterable[T]) -> T:
20
+ return tuple(sum(items, []) for items in zip(*collection)) # type: ignore
21
+
22
+
23
+ def validate_result(result: Any) -> RAW_RESULT:
24
+ try:
25
+ return parse_obj_as(list[dict[str, Any]], result)
26
+ except ValidationError as e:
27
+ raise RawObjectValidationException(f"Expected list[dict[str, Any]], Error: {e}")
28
+
29
+
30
+ def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
31
+ return (
32
+ first_entity.identifier == second_entity.identifier
33
+ and first_entity.blueprint == second_entity.blueprint
34
+ )
35
+
36
+
37
+ async def validate_integration_runtime(
38
+ port_client: PortClient,
39
+ requested_runtime: Runtime,
40
+ ) -> None:
41
+ logger.debug("Validating integration runtime")
42
+ current_integration = await port_client.get_current_integration(
43
+ should_raise=False, should_log=False
44
+ )
45
+ current_installation_type = current_integration.get("installationType", "OnPrem")
46
+ if current_integration and not requested_runtime.is_installation_type_compatible(
47
+ current_installation_type
48
+ ):
49
+ raise IntegrationRuntimeException(
50
+ f"Invalid Runtime! Requested to run existing {current_installation_type} integration in {requested_runtime} runtime."
51
+ )
52
+
53
+
54
+ Q = TypeVar("Q")
55
+
56
+
57
+ async def gather_and_split_errors_from_results(
58
+ task: Iterable[Awaitable[Q]],
59
+ result_threshold_validation: Callable[[Q | Exception], bool] | None = None,
60
+ ) -> tuple[list[Q], list[Exception]]:
61
+ valid_items: list[Q] = []
62
+ errors: list[Exception] = []
63
+ results = await asyncio.gather(*task, return_exceptions=True)
64
+ for item in results:
65
+ # return_exceptions will also catch Python BaseException which also includes KeyboardInterrupt, SystemExit, GeneratorExit
66
+ # https://docs.python.org/3/library/asyncio-task.html#asyncio.gather
67
+ # These exceptions should be raised and not caught for the application to exit properly.
68
+ # https://stackoverflow.com/a/17802352
69
+ if isinstance(item, BaseException) and not isinstance(item, Exception):
70
+ raise item
71
+ elif isinstance(item, Exception):
72
+ errors.append(item)
73
+ elif not result_threshold_validation or result_threshold_validation(item):
74
+ valid_items.append(item)
75
+
76
+ return valid_items, errors
77
+
78
+
79
+ def get_port_diff(
80
+ before: Iterable[Entity],
81
+ after: Iterable[Entity],
82
+ ) -> EntityPortDiff:
83
+ before_dict = {}
84
+ after_dict = {}
85
+ created = []
86
+ modified = []
87
+ deleted = []
88
+
89
+ # Create dictionaries for before and after lists
90
+ for entity in before:
91
+ key = (entity.identifier, entity.blueprint)
92
+ before_dict[key] = entity
93
+
94
+ for entity in after:
95
+ key = (entity.identifier, entity.blueprint)
96
+ after_dict[key] = entity
97
+
98
+ # Find created, modified, and deleted objects
99
+ for key, obj in after_dict.items():
100
+ if key not in before_dict:
101
+ created.append(obj)
102
+ else:
103
+ modified.append(obj)
104
+
105
+ for key, obj in before_dict.items():
106
+ if key not in after_dict:
107
+ deleted.append(obj)
108
+
109
+ return EntityPortDiff(created=created, modified=modified, deleted=deleted)
@@ -0,0 +1,5 @@
1
+ from port_ocean.cli.commands.main import cli_start
2
+
3
+
4
+ if __name__ == "__main__":
5
+ cli_start()
@@ -33,3 +33,7 @@ class IntegrationAlreadyStartedException(BaseOceanException):
33
33
 
34
34
  class IntegrationNotStartedException(BaseOceanException):
35
35
  pass
36
+
37
+
38
+ class IntegrationRuntimeException(BaseOceanException):
39
+ pass
@@ -6,10 +6,8 @@ class AbortDefaultCreationError(BaseOceanException):
6
6
  self,
7
7
  blueprints_to_rollback: list[str],
8
8
  errors: list[Exception],
9
- pages_to_rollback: list[str] | None = None,
10
9
  ):
11
10
  self.blueprints_to_rollback = blueprints_to_rollback
12
- self.pages_to_rollback = pages_to_rollback
13
11
  self.errors = errors
14
12
  super().__init__("Aborting defaults creation")
15
13
 
@@ -55,14 +55,14 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
55
55
  HTTPStatus.GATEWAY_TIMEOUT,
56
56
  ]
57
57
  )
58
- MAX_BACKOFF_WAIT = 60
58
+ MAX_BACKOFF_WAIT_IN_SECONDS = 60
59
59
 
60
60
  def __init__(
61
61
  self,
62
62
  wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
63
63
  max_attempts: int = 10,
64
- max_backoff_wait: float = MAX_BACKOFF_WAIT,
65
- backoff_factor: float = 0.1,
64
+ max_backoff_wait: float = MAX_BACKOFF_WAIT_IN_SECONDS,
65
+ base_delay: float = 0.1,
66
66
  jitter_ratio: float = 0.1,
67
67
  respect_retry_after_header: bool = True,
68
68
  retryable_methods: Iterable[str] | None = None,
@@ -81,7 +81,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
81
81
  max_backoff_wait (float, optional):
82
82
  The maximum amount of time (in seconds) to wait before retrying a request.
83
83
  Defaults to 60.
84
- backoff_factor (float, optional):
84
+ base_delay (float, optional):
85
85
  The factor by which the waiting time will be multiplied in each retry attempt.
86
86
  Defaults to 0.1.
87
87
  jitter_ratio (float, optional):
@@ -105,7 +105,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
105
105
  )
106
106
 
107
107
  self._max_attempts = max_attempts
108
- self._backoff_factor = backoff_factor
108
+ self._base_delay = base_delay
109
109
  self._respect_retry_after_header = respect_retry_after_header
110
110
  self._retryable_methods = (
111
111
  frozenset(retryable_methods)
@@ -132,13 +132,18 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
132
132
  httpx.Response: The response received.
133
133
 
134
134
  """
135
- transport: httpx.BaseTransport = self._wrapped_transport # type: ignore
136
- if request.method in self._retryable_methods:
137
- send_method = partial(transport.handle_request)
138
- response = self._retry_operation(request, send_method)
139
- else:
140
- response = transport.handle_request(request)
141
- return response
135
+ try:
136
+ transport: httpx.BaseTransport = self._wrapped_transport # type: ignore
137
+ if self._is_retryable_method(request):
138
+ send_method = partial(transport.handle_request)
139
+ response = self._retry_operation(request, send_method)
140
+ else:
141
+ response = transport.handle_request(request)
142
+ return response
143
+ except Exception as e:
144
+ if not self._is_retryable_method(request) and self._logger is not None:
145
+ self._logger.exception(f"{repr(e)} - {request.url}", exc_info=e)
146
+ raise e
142
147
 
143
148
  async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
144
149
  """Sends an HTTP request, possibly with retries.
@@ -150,13 +155,19 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
150
155
  The response.
151
156
 
152
157
  """
153
- transport: httpx.AsyncBaseTransport = self._wrapped_transport # type: ignore
154
- if self._is_retryable_method(request):
155
- send_method = partial(transport.handle_async_request)
156
- response = await self._retry_operation_async(request, send_method)
157
- else:
158
- response = await transport.handle_async_request(request)
159
- return response
158
+ try:
159
+ transport: httpx.AsyncBaseTransport = self._wrapped_transport # type: ignore
160
+ if self._is_retryable_method(request):
161
+ send_method = partial(transport.handle_async_request)
162
+ response = await self._retry_operation_async(request, send_method)
163
+ else:
164
+ response = await transport.handle_async_request(request)
165
+ return response
166
+ except Exception as e:
167
+ # Retyable methods are logged via _log_error
168
+ if not self._is_retryable_method(request) and self._logger is not None:
169
+ self._logger.exception(f"{repr(e)} - {request.url}", exc_info=e)
170
+ raise e
160
171
 
161
172
  async def aclose(self) -> None:
162
173
  """
@@ -179,12 +190,35 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
179
190
  transport.close()
180
191
 
181
192
  def _is_retryable_method(self, request: httpx.Request) -> bool:
182
- return request.method in self._retryable_methods
193
+ return request.method in self._retryable_methods or request.extensions.get(
194
+ "retryable", False
195
+ )
183
196
 
184
197
  def _should_retry(self, response: httpx.Response) -> bool:
185
198
  return response.status_code in self._retry_status_codes
186
199
 
187
- def _log_failure(
200
+ def _log_error(
201
+ self,
202
+ request: httpx.Request,
203
+ error: Exception | None,
204
+ ) -> None:
205
+ if not self._logger:
206
+ return
207
+
208
+ if isinstance(error, httpx.ConnectTimeout):
209
+ self._logger.error(
210
+ f"Request {request.method} {request.url} failed to connect: {str(error)}"
211
+ )
212
+ elif isinstance(error, httpx.TimeoutException):
213
+ self._logger.error(
214
+ f"Request {request.method} {request.url} failed with a timeout exception: {str(error)}"
215
+ )
216
+ elif isinstance(error, httpx.HTTPError):
217
+ self._logger.error(
218
+ f"Request {request.method} {request.url} failed with an HTTP error: {str(error)}"
219
+ )
220
+
221
+ def _log_before_retry(
188
222
  self,
189
223
  request: httpx.Request,
190
224
  sleep_time: float,
@@ -232,7 +266,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
232
266
  except ValueError:
233
267
  pass
234
268
 
235
- backoff = self._backoff_factor * (2 ** (attempts_made - 1))
269
+ backoff = self._base_delay * (2 ** (attempts_made - 1))
236
270
  jitter = (backoff * self._jitter_ratio) * random.choice([1, -1])
237
271
  total_backoff = backoff + jitter
238
272
  return min(total_backoff, self._max_backoff_wait)
@@ -249,7 +283,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
249
283
  while True:
250
284
  if attempts_made > 0:
251
285
  sleep_time = self._calculate_sleep(attempts_made, {})
252
- self._log_failure(request, sleep_time, response, error)
286
+ self._log_before_retry(request, sleep_time, response, error)
253
287
  await asyncio.sleep(sleep_time)
254
288
 
255
289
  error = None
@@ -262,9 +296,25 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
262
296
  ):
263
297
  return response
264
298
  await response.aclose()
299
+ except httpx.ConnectTimeout as e:
300
+ error = e
301
+ if remaining_attempts < 1:
302
+ self._log_error(request, error)
303
+ raise
304
+ except httpx.ReadTimeout as e:
305
+ error = e
306
+ if remaining_attempts < 1:
307
+ self._log_error(request, error)
308
+ raise
309
+ except httpx.TimeoutException as e:
310
+ error = e
311
+ if remaining_attempts < 1:
312
+ self._log_error(request, error)
313
+ raise
265
314
  except httpx.HTTPError as e:
266
315
  error = e
267
316
  if remaining_attempts < 1:
317
+ self._log_error(request, error)
268
318
  raise
269
319
  attempts_made += 1
270
320
  remaining_attempts -= 1
@@ -281,7 +331,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
281
331
  while True:
282
332
  if attempts_made > 0:
283
333
  sleep_time = self._calculate_sleep(attempts_made, {})
284
- self._log_failure(request, sleep_time, response, error)
334
+ self._log_before_retry(request, sleep_time, response, error)
285
335
  time.sleep(sleep_time)
286
336
 
287
337
  error = None
@@ -292,9 +342,20 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
292
342
  if remaining_attempts < 1 or not self._should_retry(response):
293
343
  return response
294
344
  response.close()
345
+ except httpx.ConnectTimeout as e:
346
+ error = e
347
+ if remaining_attempts < 1:
348
+ self._log_error(request, error)
349
+ raise
350
+ except httpx.TimeoutException as e:
351
+ error = e
352
+ if remaining_attempts < 1:
353
+ self._log_error(request, error)
354
+ raise
295
355
  except httpx.HTTPError as e:
296
356
  error = e
297
357
  if remaining_attempts < 1:
358
+ self._log_error(request, error)
298
359
  raise
299
360
  attempts_made += 1
300
361
  remaining_attempts -= 1
@@ -11,16 +11,22 @@ from loguru import logger
11
11
 
12
12
  from port_ocean import Ocean
13
13
  from port_ocean.context.ocean import ocean
14
+ from copy import deepcopy
15
+ from traceback import format_exception
14
16
 
15
17
 
16
18
  def _serialize_record(record: logging.LogRecord) -> dict[str, Any]:
19
+ extra = {**deepcopy(record.__dict__["extra"])}
20
+ if isinstance(extra.get("exc_info"), Exception):
21
+ serialized_exception = "".join(format_exception(extra.get("exc_info")))
22
+ extra["exc_info"] = serialized_exception
17
23
  return {
18
24
  "message": record.msg,
19
25
  "level": record.levelname,
20
26
  "timestamp": datetime.utcfromtimestamp(record.created).strftime(
21
27
  "%Y-%m-%dT%H:%M:%S.%fZ"
22
28
  ),
23
- "extra": record.__dict__["extra"],
29
+ "extra": extra,
24
30
  }
25
31
 
26
32
 
@@ -37,6 +43,7 @@ class HTTPMemoryHandler(MemoryHandler):
37
43
  self.flush_size = flush_size
38
44
  self.last_flush_time = time.time()
39
45
  self._serialized_buffer: list[dict[str, Any]] = []
46
+ self._thread_pool: list[threading.Thread] = []
40
47
 
41
48
  @property
42
49
  def ocean(self) -> Ocean | None:
@@ -46,6 +53,7 @@ class HTTPMemoryHandler(MemoryHandler):
46
53
  return None
47
54
 
48
55
  def emit(self, record: logging.LogRecord) -> None:
56
+
49
57
  self._serialized_buffer.append(_serialize_record(record))
50
58
  super().emit(record)
51
59
 
@@ -61,6 +69,11 @@ class HTTPMemoryHandler(MemoryHandler):
61
69
  return True
62
70
  return False
63
71
 
72
+ def wait_for_lingering_threads(self) -> None:
73
+ for thread in self._thread_pool:
74
+ if thread.is_alive():
75
+ thread.join()
76
+
64
77
  def flush(self) -> None:
65
78
  if self.ocean is None or not self.buffer:
66
79
  return
@@ -70,13 +83,21 @@ class HTTPMemoryHandler(MemoryHandler):
70
83
  loop.run_until_complete(self.send_logs(_ocean, logs_to_send))
71
84
  loop.close()
72
85
 
86
+ def clear_thread_pool() -> None:
87
+ for thread in self._thread_pool:
88
+ if not thread.is_alive():
89
+ self._thread_pool.remove(thread)
90
+
73
91
  self.acquire()
74
92
  logs = list(self._serialized_buffer)
75
93
  if logs:
76
94
  self.buffer.clear()
77
95
  self._serialized_buffer.clear()
78
96
  self.last_flush_time = time.time()
79
- threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs)).start()
97
+ clear_thread_pool()
98
+ thread = threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs))
99
+ thread.start()
100
+ self._thread_pool.append(thread)
80
101
  self.release()
81
102
 
82
103
  async def send_logs(
@@ -9,6 +9,7 @@ from loguru import logger
9
9
  from port_ocean.config.settings import LogLevelType
10
10
  from port_ocean.log.handlers import HTTPMemoryHandler
11
11
  from port_ocean.log.sensetive import sensitive_log_filter
12
+ from port_ocean.utils.signal import signal_handler
12
13
 
13
14
 
14
15
  def setup_logger(level: LogLevelType, enable_http_handler: bool) -> None:
@@ -35,6 +36,7 @@ def _stdout_loguru_handler(level: LogLevelType) -> None:
35
36
  diagnose=False, # hide variable values in log backtrace
36
37
  filter=sensitive_log_filter.create_filter(),
37
38
  )
39
+ logger.configure(patcher=exception_deserializer)
38
40
 
39
41
 
40
42
  def _http_loguru_handler(level: LogLevelType) -> None:
@@ -50,8 +52,13 @@ def _http_loguru_handler(level: LogLevelType) -> None:
50
52
  enqueue=True, # process logs in background
51
53
  filter=sensitive_log_filter.create_filter(full_hide=True),
52
54
  )
55
+ logger.configure(patcher=exception_deserializer)
53
56
 
54
- queue_listener = QueueListener(queue, HTTPMemoryHandler())
57
+ http_memory_handler = HTTPMemoryHandler()
58
+ signal_handler.register(http_memory_handler.wait_for_lingering_threads)
59
+ signal_handler.register(http_memory_handler.flush)
60
+
61
+ queue_listener = QueueListener(queue, http_memory_handler)
55
62
  queue_listener.start()
56
63
 
57
64
 
@@ -1,5 +1,5 @@
1
1
  import re
2
- from typing import Callable, TYPE_CHECKING
2
+ from typing import Any, Callable, TYPE_CHECKING
3
3
 
4
4
  if TYPE_CHECKING:
5
5
  from loguru import Record
@@ -21,7 +21,7 @@ secret_patterns = {
21
21
  "GitHub": r"[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]",
22
22
  "Google Cloud Platform API Key": r"AIza[0-9A-Za-z\\-_]{35}",
23
23
  "Google Cloud Platform OAuth": r"[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com",
24
- "Google (GCP) Service-account": r'"type": "service_account"',
24
+ "Google (GCP) Service-account": f'"type":{" "}"service_account"',
25
25
  "Google OAuth Access Token": r"ya29\\.[0-9A-Za-z\\-_]+",
26
26
  "Connection String": r"[a-zA-Z]+:\/\/[^/\s]+:[^/\s]+@[^/\s]+\/[^/\s]+",
27
27
  }
@@ -35,16 +35,31 @@ class SensitiveLogFilter:
35
35
  [re.compile(re.escape(token.strip())) for token in tokens if token.strip()]
36
36
  )
37
37
 
38
+ def mask_string(self, string: str, full_hide: bool = False) -> str:
39
+ masked_string = string
40
+ for pattern in self.compiled_patterns:
41
+ replace: Callable[[re.Match[str]], str] | str = (
42
+ "[REDACTED]"
43
+ if full_hide
44
+ else lambda match: match.group()[:6] + "[REDACTED]"
45
+ )
46
+ masked_string = pattern.sub(replace, masked_string)
47
+ return masked_string
48
+
49
+ def mask_object(self, obj: Any, full_hide: bool = False) -> Any:
50
+ if isinstance(obj, str):
51
+ return self.mask_string(obj, full_hide)
52
+ if isinstance(obj, list):
53
+ return [self.mask_object(o, full_hide) for o in obj]
54
+ if isinstance(obj, dict):
55
+ for k, v in obj.items():
56
+ obj[k] = self.mask_object(v, full_hide)
57
+
58
+ return obj
59
+
38
60
  def create_filter(self, full_hide: bool = False) -> Callable[["Record"], bool]:
39
61
  def _filter(record: "Record") -> bool:
40
- for pattern in self.compiled_patterns:
41
- replace: Callable[[re.Match[str]], str] | str = (
42
- "[REDACTED]"
43
- if full_hide
44
- else lambda match: match.group()[:6] + "[REDACTED]"
45
- )
46
- record["message"] = pattern.sub(replace, record["message"])
47
-
62
+ record["message"] = self.mask_string(record["message"], full_hide)
48
63
  return True
49
64
 
50
65
  return _filter
port_ocean/middlewares.py CHANGED
@@ -52,7 +52,14 @@ async def request_handler(
52
52
  request_id = generate_uuid()
53
53
 
54
54
  with logger.contextualize(request_id=request_id):
55
- logger.bind(url=str(request.url), method=request.method).info("Request started")
55
+ log_level = (
56
+ "DEBUG"
57
+ if request.url.path == "/docs" or request.url.path == "/openapi.json"
58
+ else "INFO"
59
+ )
60
+ logger.bind(url=str(request.url), method=request.method).log(
61
+ log_level, f"Request to {request.url.path} started"
62
+ )
56
63
  response = await _handle_silently(call_next, request)
57
64
 
58
65
  end_time = get_time(seconds_precision=False)
@@ -61,5 +68,6 @@ async def request_handler(
61
68
  response.headers["X-Process-Time"] = str(time_elapsed)
62
69
  logger.bind(
63
70
  time_elapsed=time_elapsed, response_status=response.status_code
64
- ).info("Request ended")
71
+ ).log(log_level, f"Request to {request.url.path} ended")
72
+
65
73
  return response
port_ocean/ocean.py CHANGED
@@ -1,13 +1,15 @@
1
1
  import asyncio
2
2
  import sys
3
3
  import threading
4
- from typing import Callable, Any, Dict
4
+ from contextlib import asynccontextmanager
5
+ from typing import Callable, Any, Dict, AsyncIterator, Type
5
6
 
6
7
  from fastapi import FastAPI, APIRouter
7
8
  from loguru import logger
8
9
  from pydantic import BaseModel
9
10
  from starlette.types import Scope, Receive, Send
10
11
 
12
+ from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
11
13
  from port_ocean.clients.port.client import PortClient
12
14
  from port_ocean.config.settings import (
13
15
  IntegrationConfiguration,
@@ -21,8 +23,9 @@ from port_ocean.core.integrations.base import BaseIntegration
21
23
  from port_ocean.log.sensetive import sensitive_log_filter
22
24
  from port_ocean.middlewares import request_handler
23
25
  from port_ocean.utils.repeat import repeat_every
24
- from port_ocean.utils.signal import init_signal_handler
26
+ from port_ocean.utils.signal import signal_handler
25
27
  from port_ocean.version import __integration_version__
28
+ from port_ocean.utils.misc import IntegrationStateStatus
26
29
 
27
30
 
28
31
  class Ocean:
@@ -31,7 +34,7 @@ class Ocean:
31
34
  app: FastAPI | None = None,
32
35
  integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
33
36
  integration_router: APIRouter | None = None,
34
- config_factory: Callable[..., BaseModel] | None = None,
37
+ config_factory: Type[BaseModel] | None = None,
35
38
  config_override: Dict[str, Any] | None = None,
36
39
  ):
37
40
  initialize_port_ocean_context(self)
@@ -39,16 +42,11 @@ class Ocean:
39
42
  self.fast_api_app.middleware("http")(request_handler)
40
43
 
41
44
  self.config = IntegrationConfiguration(
42
- base_path="./", **(config_override or {})
45
+ # type: ignore
46
+ _integration_config_model=config_factory,
47
+ **(config_override or {}),
43
48
  )
44
49
 
45
- if config_factory:
46
- raw_config = (
47
- self.config.integration.config
48
- if isinstance(self.config.integration.config, dict)
49
- else self.config.integration.config.dict()
50
- )
51
- self.config.integration.config = config_factory(**raw_config)
52
50
  # add the integration sensitive configuration to the sensitive patterns to mask out
53
51
  sensitive_log_filter.hide_sensitive_strings(
54
52
  *self.config.get_sensitive_fields_data()
@@ -67,40 +65,75 @@ class Ocean:
67
65
  integration_class(ocean) if integration_class else BaseIntegration(ocean)
68
66
  )
69
67
 
68
+ self.resync_state_updater = ResyncStateUpdater(
69
+ self.port_client, self.config.scheduled_resync_interval
70
+ )
71
+
72
+ self.app_initialized = False
73
+
74
+ def is_saas(self) -> bool:
75
+ return self.config.runtime.is_saas_runtime
76
+
70
77
  async def _setup_scheduled_resync(
71
78
  self,
72
79
  ) -> None:
73
- def execute_resync_all() -> None:
74
- loop = asyncio.new_event_loop()
75
- asyncio.set_event_loop(loop)
76
-
80
+ async def execute_resync_all() -> None:
81
+ await self.resync_state_updater.update_before_resync()
77
82
  logger.info("Starting a new scheduled resync")
78
- loop.run_until_complete(self.integration.sync_raw_all())
79
- loop.close()
83
+ try:
84
+ await self.integration.sync_raw_all()
85
+ await self.resync_state_updater.update_after_resync()
86
+ except asyncio.CancelledError:
87
+ logger.warning(
88
+ "resync was cancelled by the scheduled resync, skipping state update"
89
+ )
90
+ except Exception as e:
91
+ await self.resync_state_updater.update_after_resync(
92
+ IntegrationStateStatus.Failed
93
+ )
94
+ raise e
80
95
 
81
96
  interval = self.config.scheduled_resync_interval
97
+ loop = asyncio.get_event_loop()
82
98
  if interval is not None:
83
99
  logger.info(
84
- f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)"
100
+ f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)",
101
+ scheduled_interval=interval,
85
102
  )
86
103
  repeated_function = repeat_every(
87
104
  seconds=interval * 60,
88
105
  # Not running the resync immediately because the event listener should run resync on startup
89
106
  wait_first=True,
90
- )(lambda: threading.Thread(target=execute_resync_all).start())
107
+ )(
108
+ lambda: threading.Thread(
109
+ target=lambda: asyncio.run_coroutine_threadsafe(
110
+ execute_resync_all(), loop
111
+ )
112
+ ).start()
113
+ )
91
114
  await repeated_function()
92
115
 
93
- async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
116
+ def initialize_app(self) -> None:
94
117
  self.fast_api_app.include_router(self.integration_router, prefix="/integration")
95
118
 
96
- @self.fast_api_app.on_event("startup")
97
- async def startup() -> None:
98
- init_signal_handler()
119
+ @asynccontextmanager
120
+ async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
99
121
  try:
100
122
  await self.integration.start()
101
123
  await self._setup_scheduled_resync()
124
+ yield None
102
125
  except Exception:
103
- logger.exception("Failed to start integration")
126
+ logger.exception("Integration had a fatal error. Shutting down.")
127
+ logger.complete()
104
128
  sys.exit("Server stopped")
129
+ finally:
130
+ signal_handler.exit()
131
+
132
+ self.fast_api_app.router.lifespan_context = lifecycle
133
+ self.app_initialized = True
134
+
135
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
136
+ if not self.app_initialized:
137
+ self.initialize_app()
105
138
 
106
139
  await self.fast_api_app(scope, receive, send)