port-ocean 0.22.12__tar.gz → 0.23.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of port-ocean might be problematic. Click here for more details.

Files changed (206) hide show
  1. {port_ocean-0.22.12 → port_ocean-0.23.0}/PKG-INFO +1 -1
  2. port_ocean-0.23.0/integrations/_infra/Dockerfile.local +41 -0
  3. port_ocean-0.23.0/integrations/_infra/entry_local.sh +27 -0
  4. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/__init__.py +6 -1
  5. port_ocean-0.23.0/port_ocean/cache/base.py +25 -0
  6. port_ocean-0.23.0/port_ocean/cache/disk.py +61 -0
  7. port_ocean-0.23.0/port_ocean/cache/errors.py +10 -0
  8. port_ocean-0.23.0/port_ocean/cache/memory.py +36 -0
  9. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/settings.py +9 -2
  10. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/sync_raw.py +80 -27
  11. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/models.py +10 -0
  12. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/metric/metric.py +4 -0
  13. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/ocean.py +28 -1
  14. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/run.py +0 -1
  15. port_ocean-0.23.0/port_ocean/tests/cache/__init__.py +1 -0
  16. port_ocean-0.23.0/port_ocean/tests/cache/test_disk_cache.py +92 -0
  17. port_ocean-0.23.0/port_ocean/tests/cache/test_memory_cache.py +59 -0
  18. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/conftest.py +13 -3
  19. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +2 -28
  20. port_ocean-0.23.0/port_ocean/tests/helpers/__init__.py +0 -0
  21. port_ocean-0.23.0/port_ocean/tests/utils/test_cache.py +274 -0
  22. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/cache.py +35 -12
  23. port_ocean-0.23.0/port_ocean/utils/ipc.py +30 -0
  24. {port_ocean-0.22.12 → port_ocean-0.23.0}/pyproject.toml +4 -1
  25. port_ocean-0.22.12/port_ocean/tests/utils/test_cache.py +0 -189
  26. {port_ocean-0.22.12 → port_ocean-0.23.0}/LICENSE.md +0 -0
  27. {port_ocean-0.22.12 → port_ocean-0.23.0}/README.md +0 -0
  28. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.Deb +0 -0
  29. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.alpine +0 -0
  30. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.base.builder +0 -0
  31. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.base.runner +0 -0
  32. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.dockerignore +0 -0
  33. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Makefile +0 -0
  34. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/grpcio.sh +0 -0
  35. {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/init.sh +0 -0
  36. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/bootstrap.py +0 -0
  37. {port_ocean-0.22.12/port_ocean/cli/cookiecutter → port_ocean-0.23.0/port_ocean/cache}/__init__.py +0 -0
  38. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/__init__.py +0 -0
  39. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cli.py +0 -0
  40. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/__init__.py +0 -0
  41. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/__init___.py +0 -0
  42. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/clean.py +0 -0
  43. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/dock.py +0 -0
  44. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/group.py +0 -0
  45. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/list_integrations.py +0 -0
  46. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/main.py +0 -0
  47. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/new.py +0 -0
  48. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/pull.py +0 -0
  49. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/sail.py +0 -0
  50. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/version.py +0 -0
  51. {port_ocean-0.22.12/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests → port_ocean-0.23.0/port_ocean/cli/cookiecutter}/__init__.py +0 -0
  52. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
  53. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/extensions.py +0 -0
  54. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
  55. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
  56. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
  57. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
  58. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
  59. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
  60. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
  61. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
  62. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
  63. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
  64. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
  65. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
  66. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
  67. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
  68. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
  69. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
  70. {port_ocean-0.22.12/port_ocean/clients → port_ocean-0.23.0/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests}/__init__.py +0 -0
  71. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
  72. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/utils.py +0 -0
  73. {port_ocean-0.22.12/port_ocean/clients/auth → port_ocean-0.23.0/port_ocean/clients}/__init__.py +0 -0
  74. {port_ocean-0.22.12/port_ocean/clients/port → port_ocean-0.23.0/port_ocean/clients/auth}/__init__.py +0 -0
  75. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/auth/auth_client.py +0 -0
  76. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/auth/oauth_client.py +0 -0
  77. {port_ocean-0.22.12/port_ocean/clients/port/mixins → port_ocean-0.23.0/port_ocean/clients/port}/__init__.py +0 -0
  78. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/authentication.py +0 -0
  79. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/client.py +0 -0
  80. {port_ocean-0.22.12/port_ocean/config → port_ocean-0.23.0/port_ocean/clients/port/mixins}/__init__.py +0 -0
  81. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/blueprints.py +0 -0
  82. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/entities.py +0 -0
  83. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/integrations.py +0 -0
  84. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/migrations.py +0 -0
  85. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/organization.py +0 -0
  86. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/retry_transport.py +0 -0
  87. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/types.py +0 -0
  88. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/utils.py +0 -0
  89. {port_ocean-0.22.12/port_ocean/consumers → port_ocean-0.23.0/port_ocean/config}/__init__.py +0 -0
  90. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/base.py +0 -0
  91. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/dynamic.py +0 -0
  92. {port_ocean-0.22.12/port_ocean/context → port_ocean-0.23.0/port_ocean/consumers}/__init__.py +0 -0
  93. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/consumers/kafka_consumer.py +0 -0
  94. {port_ocean-0.22.12/port_ocean/core → port_ocean-0.23.0/port_ocean/context}/__init__.py +0 -0
  95. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/event.py +0 -0
  96. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/ocean.py +0 -0
  97. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/resource.py +0 -0
  98. {port_ocean-0.22.12/port_ocean/core/handlers/entities_state_applier/port → port_ocean-0.23.0/port_ocean/core}/__init__.py +0 -0
  99. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/__init__.py +0 -0
  100. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/clean.py +0 -0
  101. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/common.py +0 -0
  102. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/initialize.py +0 -0
  103. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/__init__.py +0 -0
  104. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/base.py +0 -0
  105. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/factory.py +0 -0
  106. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/http.py +0 -0
  107. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/kafka.py +0 -0
  108. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/once.py +0 -0
  109. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/polling.py +0 -0
  110. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/webhooks_only.py +0 -0
  111. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/__init__.py +0 -0
  112. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/base.py +0 -0
  113. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
  114. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
  115. {port_ocean-0.22.12/port_ocean/core/handlers/webhook → port_ocean-0.23.0/port_ocean/core/handlers/entities_state_applier/port}/__init__.py +0 -0
  116. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
  117. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
  118. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
  119. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
  120. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/base.py +0 -0
  121. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +0 -0
  122. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
  123. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/api.py +0 -0
  124. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/base.py +0 -0
  125. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/models.py +0 -0
  126. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/__init__.py +0 -0
  127. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
  128. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/local_queue.py +0 -0
  129. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
  130. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/resync_state_updater/updater.py +0 -0
  131. {port_ocean-0.22.12/port_ocean/core/integrations → port_ocean-0.23.0/port_ocean/core/handlers/webhook}/__init__.py +0 -0
  132. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
  133. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
  134. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
  135. {port_ocean-0.22.12/port_ocean/exceptions → port_ocean-0.23.0/port_ocean/core/integrations}/__init__.py +0 -0
  136. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/base.py +0 -0
  137. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/__init__.py +0 -0
  138. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/events.py +0 -0
  139. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/handler.py +0 -0
  140. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/live_events.py +0 -0
  141. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/sync.py +0 -0
  142. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/utils.py +0 -0
  143. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/ocean_types.py +0 -0
  144. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
  145. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/utils/utils.py +0 -0
  146. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/debug_cli.py +0 -0
  147. {port_ocean-0.22.12/port_ocean/helpers → port_ocean-0.23.0/port_ocean/exceptions}/__init__.py +0 -0
  148. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/api.py +0 -0
  149. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/base.py +0 -0
  150. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/clients.py +0 -0
  151. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/context.py +0 -0
  152. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/core.py +0 -0
  153. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/port_defaults.py +0 -0
  154. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/utils.py +0 -0
  155. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/webhook_processor.py +0 -0
  156. {port_ocean-0.22.12/port_ocean/log → port_ocean-0.23.0/port_ocean/helpers}/__init__.py +0 -0
  157. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/async_client.py +0 -0
  158. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/metric/utils.py +0 -0
  159. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/retry.py +0 -0
  160. {port_ocean-0.22.12/port_ocean/tests → port_ocean-0.23.0/port_ocean/log}/__init__.py +0 -0
  161. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/handlers.py +0 -0
  162. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/logger_setup.py +0 -0
  163. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/sensetive.py +0 -0
  164. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/middlewares.py +0 -0
  165. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/py.typed +0 -0
  166. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/sonar-project.properties +0 -0
  167. {port_ocean-0.22.12/port_ocean/tests/clients → port_ocean-0.23.0/port_ocean/tests}/__init__.py +0 -0
  168. {port_ocean-0.22.12/port_ocean/tests/clients/oauth → port_ocean-0.23.0/port_ocean/tests/clients}/__init__.py +0 -0
  169. {port_ocean-0.22.12/port_ocean/tests/helpers → port_ocean-0.23.0/port_ocean/tests/clients/oauth}/__init__.py +0 -0
  170. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
  171. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
  172. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
  173. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/conftest.py +0 -0
  174. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/defaults/test_common.py +0 -0
  175. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
  176. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +0 -0
  177. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/mixins/test_live_events.py +0 -0
  178. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
  179. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
  180. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
  181. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
  182. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +0 -0
  183. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
  184. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/test_utils.py +0 -0
  185. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
  186. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
  187. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/fake_port_api.py +0 -0
  188. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/fixtures.py +0 -0
  189. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/integration.py +0 -0
  190. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/ocean_app.py +0 -0
  191. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/port_client.py +0 -0
  192. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/smoke_test.py +0 -0
  193. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/log/test_handlers.py +0 -0
  194. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_metric.py +0 -0
  195. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_ocean.py +0 -0
  196. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_smoke.py +0 -0
  197. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/utils/test_async_iterators.py +0 -0
  198. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/__init__.py +0 -0
  199. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/async_http.py +0 -0
  200. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/async_iterators.py +0 -0
  201. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/misc.py +0 -0
  202. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/queue_utils.py +0 -0
  203. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/repeat.py +0 -0
  204. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/signal.py +0 -0
  205. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/time.py +0 -0
  206. {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: port-ocean
3
- Version: 0.22.12
3
+ Version: 0.23.0
4
4
  Summary: Port Ocean is a CLI tool for managing your Port projects.
5
5
  Home-page: https://app.getport.io
6
6
  Keywords: ocean,port-ocean,port
@@ -0,0 +1,41 @@
1
+ ARG BASE_PYTHON_IMAGE=debian:trixie-slim
2
+ # debian:trixie-slim - Python 3.12
3
+ FROM ${BASE_PYTHON_IMAGE}
4
+
5
+ RUN apt-get update \
6
+ && apt-get install -y --no-install-recommends librdkafka-dev python3 \
7
+ && apt-get clean
8
+ RUN apt-get update \
9
+ && apt-get install -y \
10
+ --no-install-recommends \
11
+ wget \
12
+ g++ \
13
+ libssl-dev \
14
+ autoconf \
15
+ automake \
16
+ libtool \
17
+ curl \
18
+ librdkafka-dev \
19
+ python3 \
20
+ python3-pip \
21
+ python3-poetry \
22
+ build-essential\
23
+ git \
24
+ python3-venv \
25
+ && apt-get clean
26
+
27
+ ARG BUILD_CONTEXT
28
+
29
+ WORKDIR /app
30
+
31
+ COPY . .
32
+ RUN rm -rf .venv-docker ${BUILD_CONTEXT}/.venv-docker
33
+ RUN python3 -m venv .venv-docker
34
+ RUN python3 -m venv ${BUILD_CONTEXT}/.venv-docker
35
+
36
+
37
+ WORKDIR /app/${BUILD_CONTEXT}
38
+
39
+ WORKDIR /app
40
+
41
+ ENTRYPOINT ["./integrations/_infra/entry_local.sh"]
@@ -0,0 +1,27 @@
1
+ #!/bin/bash
2
+ mkdir -p /tmp/prometheus_multiproc_dir
3
+ export PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc_dir
4
+ if [ -z "$BUILD_CONTEXT" ]; then
5
+ echo "BUILD_CONTEXT is not set"
6
+ exit 1
7
+ fi
8
+
9
+ if [ ! -d ".venv-docker" ]; then
10
+ /usr/bin/python3 -m venv .venv-docker
11
+ source .venv-docker/bin/activate
12
+ python -m pip install poetry
13
+ python -m poetry install
14
+ fi
15
+
16
+ cd $BUILD_CONTEXT
17
+
18
+ if [ ! -d ".venv-docker" ]; then
19
+ /usr/bin/python3 -m venv .venv-docker
20
+ source .venv-docker/bin/activate
21
+ python -m pip install poetry
22
+ python -m poetry install
23
+ fi
24
+ source .venv-docker/bin/activate
25
+ python -m pip install -e ../../
26
+
27
+ ocean sail
@@ -8,4 +8,9 @@ from .run import run # noqa: E402
8
8
  from .version import __integration_version__, __version__ # noqa: E402
9
9
 
10
10
 
11
- __all__ = ["Ocean", "run", "__version__", "__integration_version__"]
11
+ __all__ = [
12
+ "Ocean",
13
+ "run",
14
+ "__version__",
15
+ "__integration_version__",
16
+ ]
@@ -0,0 +1,25 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import Any, Optional
3
+
4
+ from port_ocean.core.models import CachingStorageMode
5
+
6
+
7
+ class CacheProvider(ABC):
8
+ """Base class for cache providers that defines the contract for all cache implementations."""
9
+
10
+ STORAGE_TYPE: CachingStorageMode
11
+
12
+ @abstractmethod
13
+ async def get(self, key: str) -> Optional[Any]:
14
+ """Get a value from the cache."""
15
+ pass
16
+
17
+ @abstractmethod
18
+ async def set(self, key: str, value: Any) -> None:
19
+ """Set a value in the cache."""
20
+ pass
21
+
22
+ @abstractmethod
23
+ async def clear(self) -> None:
24
+ """Clear all values from the cache."""
25
+ pass
@@ -0,0 +1,61 @@
1
+ import pickle
2
+ from pathlib import Path
3
+ from typing import Any, Optional
4
+
5
+ from port_ocean.cache.base import CacheProvider
6
+ from port_ocean.cache.errors import FailedToReadCacheError, FailedToWriteCacheError
7
+ from port_ocean.core.models import CachingStorageMode
8
+
9
+
10
+ class FailedToReadCacheFileError(FailedToReadCacheError):
11
+ pass
12
+
13
+
14
+ class FailedToWriteCacheFileError(FailedToWriteCacheError):
15
+ pass
16
+
17
+
18
+ class DiskCacheProvider(CacheProvider):
19
+ STORAGE_TYPE = CachingStorageMode.disk
20
+
21
+ def __init__(self, cache_dir: str | None = None) -> None:
22
+ if cache_dir is None:
23
+ cache_dir = ".ocean_cache"
24
+ self._cache_dir = Path(cache_dir)
25
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
26
+
27
+ def _get_cache_path(self, key: str) -> Path:
28
+ return self._cache_dir / f"{key}.pkl"
29
+
30
+ async def get(self, key: str) -> Optional[Any]:
31
+ cache_path = self._get_cache_path(key)
32
+ if not cache_path.exists():
33
+ return None
34
+
35
+ try:
36
+ with open(cache_path, "rb") as f:
37
+ return pickle.load(f)
38
+ except (pickle.PickleError, EOFError) as e:
39
+ raise FailedToReadCacheFileError(
40
+ f"Failed to read cache file: {cache_path}: {str(e)}"
41
+ )
42
+
43
+ async def set(self, key: str, value: Any) -> None:
44
+ cache_path = self._get_cache_path(key)
45
+ try:
46
+ with open(cache_path, "wb") as f:
47
+ pickle.dump(value, f)
48
+ except (pickle.PickleError, IOError) as e:
49
+ raise FailedToWriteCacheFileError(
50
+ f"Failed to write cache file: {cache_path}: {str(e)}"
51
+ )
52
+
53
+ async def clear(self) -> None:
54
+ try:
55
+ for cache_file in self._cache_dir.glob("*.pkl"):
56
+ try:
57
+ cache_file.unlink()
58
+ except OSError:
59
+ pass
60
+ except OSError:
61
+ pass
@@ -0,0 +1,10 @@
1
+ class CacheError(Exception):
2
+ pass
3
+
4
+
5
+ class FailedToReadCacheError(CacheError):
6
+ pass
7
+
8
+
9
+ class FailedToWriteCacheError(CacheError):
10
+ pass
@@ -0,0 +1,36 @@
1
+ from typing import Any, Optional
2
+ from port_ocean.cache.base import CacheProvider
3
+ from port_ocean.cache.errors import FailedToReadCacheError, FailedToWriteCacheError
4
+ from port_ocean.core.models import CachingStorageMode
5
+
6
+
7
+ class FailedToReadCacheMemoryError(FailedToReadCacheError):
8
+ pass
9
+
10
+
11
+ class FailedToWriteCacheMemoryError(FailedToWriteCacheError):
12
+ pass
13
+
14
+
15
+ class InMemoryCacheProvider(CacheProvider):
16
+ CACHE_KEY = "cache"
17
+ STORAGE_TYPE = CachingStorageMode.memory
18
+
19
+ def __init__(self, caching_storage: dict[str, Any] | None = None) -> None:
20
+ self._storage = caching_storage or {}
21
+ self._storage[self.CACHE_KEY] = self._storage.get(self.CACHE_KEY, {})
22
+
23
+ async def get(self, key: str) -> Optional[Any]:
24
+ try:
25
+ return self._storage.get(self.CACHE_KEY, {}).get(key)
26
+ except KeyError as e:
27
+ raise FailedToReadCacheMemoryError(f"Failed to read cache: {str(e)}")
28
+
29
+ async def set(self, key: str, value: Any) -> None:
30
+ try:
31
+ self._storage[self.CACHE_KEY][key] = value
32
+ except KeyError as e:
33
+ raise FailedToWriteCacheMemoryError(f"Failed to write cache: {str(e)}")
34
+
35
+ async def clear(self) -> None:
36
+ self._storage[self.CACHE_KEY].clear()
@@ -1,4 +1,4 @@
1
- from typing import Any, Literal, Type, cast
1
+ from typing import Any, Literal, Optional, Type, cast
2
2
 
3
3
  from pydantic import AnyHttpUrl, Extra, parse_obj_as, parse_raw_as
4
4
  from pydantic.class_validators import root_validator, validator
@@ -8,7 +8,12 @@ from pydantic.main import BaseModel
8
8
 
9
9
  from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
10
10
  from port_ocean.core.event_listener import EventListenerSettingsType
11
- from port_ocean.core.models import CreatePortResourcesOrigin, Runtime
11
+ from port_ocean.core.models import (
12
+ CachingStorageMode,
13
+ CreatePortResourcesOrigin,
14
+ Runtime,
15
+ ProcessExecutionMode,
16
+ )
12
17
  from port_ocean.utils.misc import get_integration_name, get_spec_file
13
18
 
14
19
  LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
@@ -93,6 +98,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
93
98
  )
94
99
  max_event_processing_seconds: float = 90.0
95
100
  max_wait_seconds_before_shutdown: float = 5.0
101
+ caching_storage_mode: Optional[CachingStorageMode] = Field(default=None)
102
+ process_execution_mode: Optional[ProcessExecutionMode] = Field(default=None)
96
103
 
97
104
  @validator("metrics", pre=True)
98
105
  def validate_metrics(cls, v: Any) -> MetricsSettings | dict[str, Any] | None:
@@ -1,12 +1,12 @@
1
1
  import asyncio
2
+ import uuid
2
3
  from graphlib import CycleError
3
4
  import inspect
4
5
  import typing
5
6
  from typing import Callable, Awaitable, Any
6
-
7
+ import multiprocessing
7
8
  import httpx
8
9
  from loguru import logger
9
-
10
10
  from port_ocean.clients.port.types import UserAgentType
11
11
  from port_ocean.context.event import TriggerType, event_context, EventType, event
12
12
  from port_ocean.context.ocean import ocean
@@ -20,7 +20,7 @@ from port_ocean.core.integrations.mixins.utils import (
20
20
  resync_generator_wrapper,
21
21
  resync_function_wrapper,
22
22
  )
23
- from port_ocean.core.models import Entity
23
+ from port_ocean.core.models import Entity, ProcessExecutionMode
24
24
  from port_ocean.core.ocean_types import (
25
25
  RAW_RESULT,
26
26
  RESYNC_RESULT,
@@ -33,6 +33,7 @@ from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gath
33
33
  from port_ocean.exceptions.core import OceanAbortException
34
34
  from port_ocean.helpers.metric.metric import SyncState, MetricType, MetricPhase
35
35
  from port_ocean.helpers.metric.utils import TimeMetric
36
+ from port_ocean.utils.ipc import FileIPC
36
37
 
37
38
  SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
38
39
 
@@ -267,7 +268,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
267
268
  objects_diff[0].entity_selector_diff.passed, user_agent_type
268
269
  )
269
270
 
270
-
271
271
  return CalculationResult(
272
272
  number_of_transformed_entities=len(objects_diff[0].entity_selector_diff.passed),
273
273
  entity_selector_diff=objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
@@ -344,8 +344,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
344
344
  user_agent_type,
345
345
  send_raw_data_examples_amount=send_raw_data_examples_amount
346
346
  )
347
- errors.extend(calculation_result.errors)
348
347
  passed_entities.extend(calculation_result.entity_selector_diff.passed)
348
+ errors.extend(calculation_result.errors)
349
349
  number_of_transformed_entities += calculation_result.number_of_transformed_entities
350
350
  except* OceanAbortException as error:
351
351
  ocean.metrics.sync_state = SyncState.FAILED
@@ -355,6 +355,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
355
355
  f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
356
356
  )
357
357
 
358
+
358
359
  ocean.metrics.set_metric(
359
360
  name=MetricType.SUCCESS_NAME,
360
361
  labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
@@ -584,6 +585,72 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
584
585
  for entity in event.entity_topological_sorter.get_entities(False):
585
586
  await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
586
587
 
588
+ def process_resource_in_subprocess(self,
589
+ file_ipc_map: dict[str, FileIPC],
590
+ resource: ResourceConfig,
591
+ index: int,
592
+ user_agent_type: UserAgentType,
593
+ ) -> None:
594
+ logger.info(f"process started successfully for {resource.kind} with index {index}")
595
+
596
+ async def process_resource_task() -> None:
597
+ result = await self._process_resource(
598
+ resource, index, user_agent_type
599
+ )
600
+ file_ipc_map["process_resource"].save(result)
601
+ file_ipc_map["topological_entities"].save(
602
+ event.entity_topological_sorter.entities
603
+ )
604
+
605
+ asyncio.run(process_resource_task())
606
+ logger.info(f"Process finished for {resource.kind} with index {index}")
607
+
608
+ async def process_resource(self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType) -> tuple[list[Entity], list[Exception]]:
609
+ if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
610
+ id = uuid.uuid4()
611
+ logger.info(f"Starting subprocess with id {id}")
612
+ file_ipc_map = {
613
+ "process_resource": FileIPC(id, "process_resource",([],[])),
614
+ "topological_entities": FileIPC(id, "topological_entities",[]),
615
+ }
616
+ process = multiprocessing.Process(target=self.process_resource_in_subprocess, args=(file_ipc_map,resource,index,user_agent_type))
617
+ process.start()
618
+ process.join()
619
+ if process.exitcode != 0:
620
+ logger.error(f"Process {id} failed with exit code {process.exitcode}")
621
+ event.entity_topological_sorter.entities.extend(file_ipc_map["topological_entities"].load())
622
+ return file_ipc_map["process_resource"].load()
623
+
624
+ else:
625
+ return await self._process_resource(resource,index,user_agent_type)
626
+
627
+ async def _process_resource(self,resource: ResourceConfig, index: int, user_agent_type: UserAgentType)-> tuple[list[Entity], list[Exception]]:
628
+ # create resource context per resource kind, so resync method could have access to the resource
629
+ # config as we might have multiple resources in the same event
630
+ async with resource_context(resource,index):
631
+ resource_kind_id = f"{resource.kind}-{index}"
632
+ ocean.metrics.sync_state = SyncState.SYNCING
633
+ task = asyncio.create_task(
634
+ self._register_in_batches(resource, user_agent_type)
635
+ )
636
+ event.on_abort(lambda: task.cancel())
637
+ kind_results: tuple[list[Entity], list[Exception]] = await task
638
+ ocean.metrics.set_metric(
639
+ name=MetricType.OBJECT_COUNT_NAME,
640
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
641
+ value=len(kind_results[0])
642
+ )
643
+
644
+ if ocean.metrics.sync_state != SyncState.FAILED:
645
+ ocean.metrics.sync_state = SyncState.COMPLETED
646
+
647
+ await ocean.metrics.send_metrics_to_webhook(
648
+ kind=resource_kind_id
649
+ )
650
+ # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
651
+
652
+ return kind_results
653
+
587
654
  @TimeMetric(MetricPhase.RESYNC)
588
655
  async def sync_raw_all(
589
656
  self,
@@ -622,6 +689,9 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
622
689
  ocean.metrics.initialize_metrics(kinds)
623
690
  # await ocean.metrics.report_sync_metrics(kinds=kinds) # TODO: uncomment this when end points are ready
624
691
 
692
+ # Clear cache
693
+ await ocean.app.cache_provider.clear()
694
+
625
695
  # Execute resync_start hooks
626
696
  for resync_start_fn in self.event_strategy["resync_start"]:
627
697
  await resync_start_fn()
@@ -640,32 +710,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
640
710
 
641
711
  creation_results: list[tuple[list[Entity], list[Exception]]] = []
642
712
 
643
-
713
+ multiprocessing.set_start_method('fork', True)
644
714
  try:
645
715
  for index,resource in enumerate(app_config.resources):
646
- # create resource context per resource kind, so resync method could have access to the resource
647
- # config as we might have multiple resources in the same event
648
- async with resource_context(resource,index):
649
- resource_kind_id = f"{resource.kind}-{index}"
650
- ocean.metrics.sync_state = SyncState.SYNCING
651
- # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
652
-
653
- task = asyncio.create_task(
654
- self._register_in_batches(resource, user_agent_type)
655
- )
656
716
 
657
- event.on_abort(lambda: task.cancel())
658
- kind_results: tuple[list[Entity], list[Exception]] = await task
717
+ logger.info(f"Starting processing resource {resource.kind} with index {index}")
659
718
 
660
- creation_results.append(kind_results)
661
-
662
- if ocean.metrics.sync_state != SyncState.FAILED:
663
- ocean.metrics.sync_state = SyncState.COMPLETED
664
-
665
- await ocean.metrics.send_metrics_to_webhook(
666
- kind=resource_kind_id
667
- )
668
- # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
719
+ creation_results.append(await self.process_resource(resource,index,user_agent_type))
669
720
 
670
721
  await self.sort_and_upsert_failed_entities(user_agent_type)
671
722
 
@@ -721,3 +772,5 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
721
772
  logger.info("Finished executing resync_complete hooks")
722
773
 
723
774
  return True
775
+ finally:
776
+ await ocean.app.cache_provider.clear()
@@ -11,6 +11,16 @@ class CreatePortResourcesOrigin(StrEnum):
11
11
  Port = "Port"
12
12
 
13
13
 
14
+ class ProcessExecutionMode(StrEnum):
15
+ multi_process = "multi_process"
16
+ single_process = "single_process"
17
+
18
+
19
+ class CachingStorageMode(StrEnum):
20
+ disk = "disk"
21
+ memory = "memory"
22
+
23
+
14
24
  class Runtime(Enum):
15
25
  Saas = "Saas"
16
26
  OnPrem = "OnPrem"
@@ -10,6 +10,7 @@ from prometheus_client import Gauge
10
10
  import prometheus_client.openmetrics
11
11
  import prometheus_client.openmetrics.exposition
12
12
  import prometheus_client.parser
13
+ from prometheus_client import multiprocess
13
14
 
14
15
  if TYPE_CHECKING:
15
16
  from port_ocean.config.settings import MetricsSettings, IntegrationSettings
@@ -108,11 +109,14 @@ class Metrics:
108
109
  metrics_settings: "MetricsSettings",
109
110
  integration_configuration: "IntegrationSettings",
110
111
  port_client: "PortClient",
112
+ multiprocessing_enabled: bool = False,
111
113
  ) -> None:
112
114
  self.metrics_settings = metrics_settings
113
115
  self.integration_configuration = integration_configuration
114
116
  self.port_client = port_client
115
117
  self.registry = prometheus_client.CollectorRegistry()
118
+ if multiprocessing_enabled:
119
+ multiprocess.MultiProcessCollector(self.registry)
116
120
  self.metrics: dict[str, Gauge] = {}
117
121
  self.load_metrics()
118
122
  self._integration_version: Optional[str] = None
@@ -4,6 +4,10 @@ from contextlib import asynccontextmanager
4
4
  import threading
5
5
  from typing import Any, AsyncIterator, Callable, Dict, Type
6
6
 
7
+ from port_ocean.cache.base import CacheProvider
8
+ from port_ocean.cache.disk import DiskCacheProvider
9
+ from port_ocean.cache.memory import InMemoryCacheProvider
10
+ from port_ocean.core.models import ProcessExecutionMode
7
11
  import port_ocean.helpers.metric.metric
8
12
 
9
13
  from fastapi import FastAPI, APIRouter
@@ -66,11 +70,16 @@ class Ocean:
66
70
  integration_type=self.config.integration.type,
67
71
  integration_version=__integration_version__,
68
72
  )
69
-
73
+ self.cache_provider: CacheProvider = self._get_caching_provider()
74
+ self.process_execution_mode: ProcessExecutionMode = (
75
+ self._get_process_execution_mode()
76
+ )
70
77
  self.metrics = port_ocean.helpers.metric.metric.Metrics(
71
78
  metrics_settings=self.config.metrics,
72
79
  integration_configuration=self.config.integration,
73
80
  port_client=self.port_client,
81
+ multiprocessing_enabled=self.process_execution_mode
82
+ == ProcessExecutionMode.multi_process,
74
83
  )
75
84
 
76
85
  self.webhook_manager = LiveEventsProcessorManager(
@@ -90,6 +99,24 @@ class Ocean:
90
99
 
91
100
  self.app_initialized = False
92
101
 
102
+ def _get_process_execution_mode(self) -> ProcessExecutionMode:
103
+ if self.config.process_execution_mode:
104
+ return self.config.process_execution_mode
105
+ return ProcessExecutionMode.single_process
106
+
107
+ def _get_caching_provider(self) -> CacheProvider:
108
+ if self.config.caching_storage_mode:
109
+ caching_type_to_provider = {
110
+ DiskCacheProvider.STORAGE_TYPE: DiskCacheProvider,
111
+ InMemoryCacheProvider.STORAGE_TYPE: InMemoryCacheProvider,
112
+ }
113
+ if self.config.caching_storage_mode in caching_type_to_provider:
114
+ return caching_type_to_provider[self.config.caching_storage_mode]()
115
+
116
+ if self.config.process_execution_mode == ProcessExecutionMode.multi_process:
117
+ return DiskCacheProvider()
118
+ return InMemoryCacheProvider()
119
+
93
120
  def is_saas(self) -> bool:
94
121
  return self.config.runtime.is_saas_runtime
95
122
 
@@ -21,7 +21,6 @@ def _get_default_config_factory() -> None | Type[BaseModel]:
21
21
  config_factory = None
22
22
  if spec is not None:
23
23
  config_factory = default_config_factory(spec.get("configurations", []))
24
-
25
24
  return config_factory
26
25
 
27
26
 
@@ -0,0 +1 @@
1
+ """Tests for cache providers."""
@@ -0,0 +1,92 @@
1
+ import os
2
+ import pytest
3
+ from pathlib import Path
4
+
5
+ from port_ocean.cache.disk import (
6
+ DiskCacheProvider,
7
+ FailedToReadCacheFileError,
8
+ FailedToWriteCacheFileError,
9
+ )
10
+
11
+
12
+ @pytest.fixture
13
+ def disk_cache(tmp_path: Path) -> DiskCacheProvider:
14
+ """Fixture that provides a DiskCacheProvider with a temporary directory."""
15
+ return DiskCacheProvider(cache_dir=str(tmp_path))
16
+
17
+
18
+ @pytest.mark.asyncio
19
+ async def test_disk_cache_set_get(disk_cache: DiskCacheProvider) -> None:
20
+ """Test setting and getting values from disk cache."""
21
+ # Test basic set/get
22
+ await disk_cache.set("test_key", "test_value")
23
+ assert await disk_cache.get("test_key") == "test_value"
24
+
25
+ # Test with different types
26
+ test_data = {
27
+ "string": "hello",
28
+ "int": 42,
29
+ "float": 3.14,
30
+ "list": [1, 2, 3],
31
+ "dict": {"a": 1, "b": 2},
32
+ }
33
+
34
+ for key, value in test_data.items():
35
+ await disk_cache.set(key, value)
36
+ assert await disk_cache.get(key) == value
37
+
38
+
39
+ @pytest.mark.asyncio
40
+ async def test_disk_cache_clear(disk_cache: DiskCacheProvider) -> None:
41
+ """Test clearing all values from disk cache."""
42
+ # Add multiple values
43
+ for i in range(5):
44
+ await disk_cache.set(f"key_{i}", f"value_{i}")
45
+
46
+ # Verify values exist
47
+ for i in range(5):
48
+ assert await disk_cache.get(f"key_{i}") == f"value_{i}"
49
+
50
+ # Clear cache
51
+ await disk_cache.clear()
52
+
53
+ # Verify all values are gone
54
+ for i in range(5):
55
+ assert await disk_cache.get(f"key_{i}") is None
56
+
57
+
58
+ @pytest.mark.asyncio
59
+ async def test_disk_cache_nonexistent_key(disk_cache: DiskCacheProvider) -> None:
60
+ """Test getting a nonexistent key from disk cache."""
61
+ assert await disk_cache.get("nonexistent_key") is None
62
+
63
+
64
+ @pytest.mark.asyncio
65
+ async def test_disk_cache_corrupted_file(
66
+ disk_cache: DiskCacheProvider, tmp_path: Path
67
+ ) -> None:
68
+ """Test handling of corrupted cache files."""
69
+ # Create a corrupted pickle file
70
+ cache_path = tmp_path / "test_key.pkl"
71
+ with open(cache_path, "wb") as f:
72
+ f.write(b"invalid pickle data")
73
+
74
+ # Attempting to read should raise FailedToReadCacheFileError
75
+ with pytest.raises(FailedToReadCacheFileError):
76
+ await disk_cache.get("test_key")
77
+
78
+
79
+ @pytest.mark.asyncio
80
+ async def test_disk_cache_write_error(
81
+ disk_cache: DiskCacheProvider, tmp_path: Path
82
+ ) -> None:
83
+ """Test handling of write errors."""
84
+ # Make the cache directory read-only
85
+ os.chmod(tmp_path, 0o444)
86
+
87
+ # Attempting to write should raise FailedToWriteCacheFileError
88
+ with pytest.raises(FailedToWriteCacheFileError):
89
+ await disk_cache.set("test_key", "test_value")
90
+
91
+ # Restore permissions
92
+ os.chmod(tmp_path, 0o755)