port-ocean 0.24.22__tar.gz → 0.25.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. {port_ocean-0.24.22 → port_ocean-0.25.0}/PKG-INFO +1 -1
  2. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/sync_raw.py +307 -117
  3. {port_ocean-0.24.22 → port_ocean-0.25.0}/pyproject.toml +1 -1
  4. {port_ocean-0.24.22 → port_ocean-0.25.0}/LICENSE.md +0 -0
  5. {port_ocean-0.24.22 → port_ocean-0.25.0}/README.md +0 -0
  6. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.Deb +0 -0
  7. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.alpine +0 -0
  8. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.base.builder +0 -0
  9. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.base.runner +0 -0
  10. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.dockerignore +0 -0
  11. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.local +0 -0
  12. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Makefile +0 -0
  13. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/entry_local.sh +0 -0
  14. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/grpcio.sh +0 -0
  15. {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/init.sh +0 -0
  16. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/__init__.py +0 -0
  17. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/bootstrap.py +0 -0
  18. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/__init__.py +0 -0
  19. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/base.py +0 -0
  20. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/disk.py +0 -0
  21. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/errors.py +0 -0
  22. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/memory.py +0 -0
  23. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/__init__.py +0 -0
  24. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cli.py +0 -0
  25. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/__init__.py +0 -0
  26. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/__init___.py +0 -0
  27. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/clean.py +0 -0
  28. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/dock.py +0 -0
  29. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/group.py +0 -0
  30. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/list_integrations.py +0 -0
  31. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/main.py +0 -0
  32. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/new.py +0 -0
  33. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/pull.py +0 -0
  34. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/sail.py +0 -0
  35. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/version.py +0 -0
  36. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/__init__.py +0 -0
  37. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
  38. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/extensions.py +0 -0
  39. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
  40. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
  41. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
  42. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
  43. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
  44. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
  45. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
  46. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
  47. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
  48. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
  49. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
  50. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
  51. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
  52. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
  53. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
  54. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
  55. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py +0 -0
  56. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
  57. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/utils.py +0 -0
  58. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/__init__.py +0 -0
  59. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/__init__.py +0 -0
  60. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/auth_client.py +0 -0
  61. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/oauth_client.py +0 -0
  62. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/__init__.py +0 -0
  63. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/authentication.py +0 -0
  64. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/client.py +0 -0
  65. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/__init__.py +0 -0
  66. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/blueprints.py +0 -0
  67. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/entities.py +0 -0
  68. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/integrations.py +0 -0
  69. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/migrations.py +0 -0
  70. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/organization.py +0 -0
  71. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/retry_transport.py +0 -0
  72. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/types.py +0 -0
  73. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/utils.py +0 -0
  74. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/__init__.py +0 -0
  75. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/base.py +0 -0
  76. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/dynamic.py +0 -0
  77. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/settings.py +0 -0
  78. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/consumers/__init__.py +0 -0
  79. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/consumers/kafka_consumer.py +0 -0
  80. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/__init__.py +0 -0
  81. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/event.py +0 -0
  82. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/metric_resource.py +0 -0
  83. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/ocean.py +0 -0
  84. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/resource.py +0 -0
  85. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/__init__.py +0 -0
  86. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/__init__.py +0 -0
  87. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/clean.py +0 -0
  88. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/common.py +0 -0
  89. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/initialize.py +0 -0
  90. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/__init__.py +0 -0
  91. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/base.py +0 -0
  92. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/factory.py +0 -0
  93. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/http.py +0 -0
  94. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/kafka.py +0 -0
  95. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/once.py +0 -0
  96. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/polling.py +0 -0
  97. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/webhooks_only.py +0 -0
  98. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/__init__.py +0 -0
  99. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/base.py +0 -0
  100. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
  101. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
  102. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/__init__.py +0 -0
  103. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
  104. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
  105. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
  106. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
  107. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/base.py +0 -0
  108. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +0 -0
  109. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
  110. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/api.py +0 -0
  111. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/base.py +0 -0
  112. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/models.py +0 -0
  113. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/__init__.py +0 -0
  114. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
  115. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/local_queue.py +0 -0
  116. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
  117. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/resync_state_updater/updater.py +0 -0
  118. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/__init__.py +0 -0
  119. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
  120. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
  121. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
  122. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/__init__.py +0 -0
  123. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/base.py +0 -0
  124. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/__init__.py +0 -0
  125. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/events.py +0 -0
  126. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/handler.py +0 -0
  127. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/live_events.py +0 -0
  128. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/sync.py +0 -0
  129. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/utils.py +0 -0
  130. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/models.py +0 -0
  131. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/ocean_types.py +0 -0
  132. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
  133. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/utils/utils.py +0 -0
  134. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/debug_cli.py +0 -0
  135. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/__init__.py +0 -0
  136. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/api.py +0 -0
  137. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/base.py +0 -0
  138. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/clients.py +0 -0
  139. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/context.py +0 -0
  140. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/core.py +0 -0
  141. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/port_defaults.py +0 -0
  142. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/utils.py +0 -0
  143. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/webhook_processor.py +0 -0
  144. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/__init__.py +0 -0
  145. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/async_client.py +0 -0
  146. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/metric/metric.py +0 -0
  147. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/metric/utils.py +0 -0
  148. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/retry.py +0 -0
  149. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/__init__.py +0 -0
  150. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/handlers.py +0 -0
  151. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/logger_setup.py +0 -0
  152. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/sensetive.py +0 -0
  153. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/middlewares.py +0 -0
  154. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/ocean.py +0 -0
  155. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/py.typed +0 -0
  156. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/run.py +0 -0
  157. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/sonar-project.properties +0 -0
  158. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/__init__.py +0 -0
  159. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/__init__.py +0 -0
  160. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/test_disk_cache.py +0 -0
  161. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/test_memory_cache.py +0 -0
  162. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/__init__.py +0 -0
  163. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/oauth/__init__.py +0 -0
  164. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
  165. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
  166. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_integrations.py +0 -0
  167. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
  168. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/config/test_config.py +0 -0
  169. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/conftest.py +0 -0
  170. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/conftest.py +0 -0
  171. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/defaults/test_common.py +0 -0
  172. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
  173. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +0 -0
  174. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/mixins/test_live_events.py +0 -0
  175. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +0 -0
  176. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
  177. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
  178. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
  179. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
  180. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +0 -0
  181. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
  182. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/test_utils.py +0 -0
  183. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
  184. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
  185. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/__init__.py +0 -0
  186. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/fake_port_api.py +0 -0
  187. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/fixtures.py +0 -0
  188. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/integration.py +0 -0
  189. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/ocean_app.py +0 -0
  190. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/port_client.py +0 -0
  191. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/smoke_test.py +0 -0
  192. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/log/test_handlers.py +0 -0
  193. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_metric.py +0 -0
  194. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_ocean.py +0 -0
  195. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_smoke.py +0 -0
  196. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/utils/test_async_iterators.py +0 -0
  197. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/utils/test_cache.py +0 -0
  198. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/__init__.py +0 -0
  199. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/async_http.py +0 -0
  200. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/async_iterators.py +0 -0
  201. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/cache.py +0 -0
  202. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/ipc.py +0 -0
  203. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/misc.py +0 -0
  204. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/queue_utils.py +0 -0
  205. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/repeat.py +0 -0
  206. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/signal.py +0 -0
  207. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/time.py +0 -0
  208. {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: port-ocean
3
- Version: 0.24.22
3
+ Version: 0.25.0
4
4
  Summary: Port Ocean is a CLI tool for managing your Port projects.
5
5
  Home-page: https://app.getport.io
6
6
  Keywords: ocean,port-ocean,port
@@ -32,9 +32,21 @@ from port_ocean.core.ocean_types import (
32
32
  RAW_ITEM,
33
33
  CalculationResult,
34
34
  )
35
- from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
36
- from port_ocean.exceptions.core import IntegrationSubProcessFailedException, OceanAbortException
37
- from port_ocean.helpers.metric.metric import MetricResourceKind, SyncState, MetricType, MetricPhase
35
+ from port_ocean.core.utils.utils import (
36
+ resolve_entities_diff,
37
+ zip_and_sum,
38
+ gather_and_split_errors_from_results,
39
+ )
40
+ from port_ocean.exceptions.core import (
41
+ IntegrationSubProcessFailedException,
42
+ OceanAbortException,
43
+ )
44
+ from port_ocean.helpers.metric.metric import (
45
+ MetricResourceKind,
46
+ SyncState,
47
+ MetricType,
48
+ MetricPhase,
49
+ )
38
50
  from port_ocean.helpers.metric.utils import TimeMetric, TimeMetricWithResourceKind
39
51
  from port_ocean.utils.ipc import FileIPC
40
52
 
@@ -101,10 +113,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
101
113
  results = []
102
114
  for task in fns:
103
115
  if inspect.isasyncgenfunction(task):
104
- logger.info(f"Found async generator function for {resource_config.kind} name: {task.__qualname__}")
116
+ logger.info(
117
+ f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
118
+ )
105
119
  results.append(resync_generator_wrapper(task, resource_config.kind))
106
120
  else:
107
- logger.info(f"Found sync function for {resource_config.kind} name: {task.__qualname__}")
121
+ logger.info(
122
+ f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
123
+ )
108
124
  task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
109
125
  tasks.append(resync_function_wrapper(task, resource_config.kind))
110
126
 
@@ -155,14 +171,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
155
171
  {
156
172
  "property": "$identifier",
157
173
  "operator": "in",
158
- "value": [entity.identifier for entity in entities]
174
+ "value": [entity.identifier for entity in entities],
159
175
  },
160
176
  {
161
177
  "property": "$blueprint",
162
178
  "operator": "=",
163
- "value": entities[0].blueprint
164
- }
165
- ]
179
+ "value": entities[0].blueprint,
180
+ },
181
+ ],
166
182
  }
167
183
 
168
184
  async def _map_entities_compared_with_port(
@@ -174,7 +190,10 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
174
190
  if not entities:
175
191
  return []
176
192
 
177
- if entities[0].is_using_search_identifier or entities[0].is_using_search_relation:
193
+ if (
194
+ entities[0].is_using_search_identifier
195
+ or entities[0].is_using_search_relation
196
+ ):
178
197
  return entities
179
198
 
180
199
  MIN_ENTITIES_TO_MAP = 10
@@ -186,21 +205,21 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
186
205
 
187
206
  # Process entities in batches
188
207
  for start_index in range(0, len(entities), BATCH_SIZE):
189
- entities_batch = entities[start_index:start_index + BATCH_SIZE]
208
+ entities_batch = entities[start_index : start_index + BATCH_SIZE]
190
209
  batch_results = await self._fetch_entities_batch_from_port(
191
- entities_batch,
192
- resource,
193
- user_agent_type
210
+ entities_batch, resource, user_agent_type
194
211
  )
195
212
  entities_at_port_with_properties.extend(batch_results)
196
213
 
197
- logger.info("Got entities from port with properties and relations", port_entities=len(entities_at_port_with_properties))
214
+ logger.info(
215
+ "Got entities from port with properties and relations",
216
+ port_entities=len(entities_at_port_with_properties),
217
+ )
198
218
 
199
219
  if len(entities_at_port_with_properties) > 0:
200
220
  return resolve_entities_diff(entities, entities_at_port_with_properties)
201
221
  return entities
202
222
 
203
-
204
223
  async def _fetch_entities_batch_from_port(
205
224
  self,
206
225
  entities_batch: list[Entity],
@@ -210,16 +229,18 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
210
229
  query = self._construct_search_query_for_entities(entities_batch)
211
230
  return await ocean.port_client.search_entities(
212
231
  user_agent_type,
213
- parameters_to_include=["blueprint", "identifier"] + (
214
- ["title"] if resource.port.entity.mappings.title != None else []
215
- ) + (
216
- ["team"] if resource.port.entity.mappings.team != None else []
217
- ) + [
218
- f"properties.{prop}" for prop in resource.port.entity.mappings.properties
219
- ] + [
220
- f"relations.{relation}" for relation in resource.port.entity.mappings.relations
232
+ parameters_to_include=["blueprint", "identifier"]
233
+ + (["title"] if resource.port.entity.mappings.title != None else [])
234
+ + (["team"] if resource.port.entity.mappings.team != None else [])
235
+ + [
236
+ f"properties.{prop}"
237
+ for prop in resource.port.entity.mappings.properties
238
+ ]
239
+ + [
240
+ f"relations.{relation}"
241
+ for relation in resource.port.entity.mappings.relations
221
242
  ],
222
- query=query
243
+ query=query,
223
244
  )
224
245
 
225
246
  async def _register_resource_raw(
@@ -228,7 +249,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
228
249
  results: list[dict[Any, Any]],
229
250
  user_agent_type: UserAgentType,
230
251
  parse_all: bool = False,
231
- send_raw_data_examples_amount: int = 0
252
+ send_raw_data_examples_amount: int = 0,
232
253
  ) -> CalculationResult:
233
254
  objects_diff = await self._calculate_raw(
234
255
  [(resource, results)], parse_all, send_raw_data_examples_amount
@@ -236,8 +257,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
236
257
 
237
258
  ocean.metrics.inc_metric(
238
259
  name=MetricType.OBJECT_COUNT_NAME,
239
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM, MetricPhase.TransformResult.FAILED],
240
- value=len(objects_diff[0].entity_selector_diff.failed)
260
+ labels=[
261
+ ocean.metrics.current_resource_kind(),
262
+ MetricPhase.TRANSFORM,
263
+ MetricPhase.TransformResult.FAILED,
264
+ ],
265
+ value=len(objects_diff[0].entity_selector_diff.failed),
241
266
  )
242
267
 
243
268
  modified_objects = []
@@ -247,43 +272,67 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
247
272
  changed_entities = await self._map_entities_compared_with_port(
248
273
  objects_diff[0].entity_selector_diff.passed,
249
274
  resource,
250
- user_agent_type
275
+ user_agent_type,
251
276
  )
252
277
  if changed_entities:
253
- logger.info("Upserting changed entities", changed_entities=len(changed_entities),
254
- total_entities=len(objects_diff[0].entity_selector_diff.passed))
278
+ logger.info(
279
+ "Upserting changed entities",
280
+ changed_entities=len(changed_entities),
281
+ total_entities=len(objects_diff[0].entity_selector_diff.passed),
282
+ )
255
283
  ocean.metrics.inc_metric(
256
- name=MetricType.OBJECT_COUNT_NAME,
257
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
258
- value=len(objects_diff[0].entity_selector_diff.passed) - len(changed_entities)
259
- )
284
+ name=MetricType.OBJECT_COUNT_NAME,
285
+ labels=[
286
+ ocean.metrics.current_resource_kind(),
287
+ MetricPhase.LOAD,
288
+ MetricPhase.LoadResult.SKIPPED,
289
+ ],
290
+ value=len(objects_diff[0].entity_selector_diff.passed)
291
+ - len(changed_entities),
292
+ )
260
293
  await self.entities_state_applier.upsert(
261
294
  changed_entities, user_agent_type
262
295
  )
263
296
 
264
297
  else:
265
- logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
298
+ logger.info(
299
+ "Entities in batch didn't changed since last sync, skipping",
300
+ total_entities=len(objects_diff[0].entity_selector_diff.passed),
301
+ )
266
302
  ocean.metrics.inc_metric(
267
- name=MetricType.OBJECT_COUNT_NAME,
268
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
269
- value=len(objects_diff[0].entity_selector_diff.passed)
270
- )
271
- modified_objects = [ocean.port_client._reduce_entity(entity) for entity in objects_diff[0].entity_selector_diff.passed]
303
+ name=MetricType.OBJECT_COUNT_NAME,
304
+ labels=[
305
+ ocean.metrics.current_resource_kind(),
306
+ MetricPhase.LOAD,
307
+ MetricPhase.LoadResult.SKIPPED,
308
+ ],
309
+ value=len(objects_diff[0].entity_selector_diff.passed),
310
+ )
311
+ modified_objects = [
312
+ ocean.port_client._reduce_entity(entity)
313
+ for entity in objects_diff[0].entity_selector_diff.passed
314
+ ]
272
315
  except Exception as e:
273
- logger.warning(f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}")
316
+ logger.warning(
317
+ f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}"
318
+ )
274
319
  modified_objects = await self.entities_state_applier.upsert(
275
320
  objects_diff[0].entity_selector_diff.passed, user_agent_type
276
- )
321
+ )
277
322
  else:
278
- modified_objects = await self.entities_state_applier.upsert(
279
- objects_diff[0].entity_selector_diff.passed, user_agent_type
280
- )
323
+ modified_objects = await self.entities_state_applier.upsert(
324
+ objects_diff[0].entity_selector_diff.passed, user_agent_type
325
+ )
281
326
 
282
327
  return CalculationResult(
283
- number_of_transformed_entities=len(objects_diff[0].entity_selector_diff.passed),
284
- entity_selector_diff=objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
328
+ number_of_transformed_entities=len(
329
+ objects_diff[0].entity_selector_diff.passed
330
+ ),
331
+ entity_selector_diff=objects_diff[0].entity_selector_diff._replace(
332
+ passed=modified_objects
333
+ ),
285
334
  errors=objects_diff[0].errors,
286
- misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys
335
+ misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys,
287
336
  )
288
337
 
289
338
  async def _unregister_resource_raw(
@@ -334,17 +383,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
334
383
  resource_config,
335
384
  raw_results,
336
385
  user_agent_type,
337
- send_raw_data_examples_amount=send_raw_data_examples_amount
386
+ send_raw_data_examples_amount=send_raw_data_examples_amount,
338
387
  )
339
388
  errors.extend(calculation_result.errors)
340
389
  passed_entities = list(calculation_result.entity_selector_diff.passed)
341
- number_of_transformed_entities += calculation_result.number_of_transformed_entities
390
+ number_of_transformed_entities += (
391
+ calculation_result.number_of_transformed_entities
392
+ )
342
393
  logger.info(
343
394
  f"Finished registering change for {len(raw_results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
344
395
  )
345
396
 
346
-
347
-
348
397
  for generator in async_generators:
349
398
  try:
350
399
  async for items in generator:
@@ -358,11 +407,15 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
358
407
  resource_config,
359
408
  items,
360
409
  user_agent_type,
361
- send_raw_data_examples_amount=send_raw_data_examples_amount
410
+ send_raw_data_examples_amount=send_raw_data_examples_amount,
411
+ )
412
+ passed_entities.extend(
413
+ calculation_result.entity_selector_diff.passed
362
414
  )
363
- passed_entities.extend(calculation_result.entity_selector_diff.passed)
364
415
  errors.extend(calculation_result.errors)
365
- number_of_transformed_entities += calculation_result.number_of_transformed_entities
416
+ number_of_transformed_entities += (
417
+ calculation_result.number_of_transformed_entities
418
+ )
366
419
  except* OceanAbortException as error:
367
420
  ocean.metrics.sync_state = SyncState.FAILED
368
421
  errors.append(error)
@@ -371,29 +424,40 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
371
424
  f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
372
425
  )
373
426
 
374
-
375
427
  ocean.metrics.set_metric(
376
428
  name=MetricType.SUCCESS_NAME,
377
429
  labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
378
- value=int(not errors)
430
+ value=int(not errors),
379
431
  )
380
432
 
381
433
  ocean.metrics.inc_metric(
382
434
  name=MetricType.OBJECT_COUNT_NAME,
383
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.EXTRACTED],
384
- value=number_of_raw_results
435
+ labels=[
436
+ ocean.metrics.current_resource_kind(),
437
+ MetricPhase.EXTRACT,
438
+ MetricPhase.ExtractResult.EXTRACTED,
439
+ ],
440
+ value=number_of_raw_results,
385
441
  )
386
442
 
387
443
  ocean.metrics.inc_metric(
388
444
  name=MetricType.OBJECT_COUNT_NAME,
389
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.TRANSFORMED],
390
- value=number_of_transformed_entities
445
+ labels=[
446
+ ocean.metrics.current_resource_kind(),
447
+ MetricPhase.TRANSFORM,
448
+ MetricPhase.TransformResult.TRANSFORMED,
449
+ ],
450
+ value=number_of_transformed_entities,
391
451
  )
392
452
 
393
453
  ocean.metrics.inc_metric(
394
454
  name=MetricType.OBJECT_COUNT_NAME,
395
- labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.FILTERED_OUT],
396
- value=number_of_raw_results -number_of_transformed_entities
455
+ labels=[
456
+ ocean.metrics.current_resource_kind(),
457
+ MetricPhase.TRANSFORM,
458
+ MetricPhase.TransformResult.FILTERED_OUT,
459
+ ],
460
+ value=number_of_raw_results - number_of_transformed_entities,
397
461
  )
398
462
 
399
463
  return passed_entities, errors
@@ -440,7 +504,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
440
504
  errors = sum(errors, [])
441
505
  misconfigured_entity_keys = list(misconfigured_entity_keys)
442
506
 
443
-
444
507
  if errors:
445
508
  message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
446
509
  logger.error(message, exc_info=errors)
@@ -580,34 +643,54 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
580
643
  user_agent_type,
581
644
  )
582
645
 
583
- async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
646
+ async def sort_and_upsert_failed_entities(
647
+ self, user_agent_type: UserAgentType
648
+ ) -> None:
584
649
  try:
585
650
  if not event.entity_topological_sorter.should_execute():
586
651
  return None
587
- logger.info(f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count())
652
+ logger.info(
653
+ f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",
654
+ failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count(),
655
+ )
588
656
 
589
657
  for entity in event.entity_topological_sorter.get_entities():
590
- await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
658
+ await self.entities_state_applier.context.port_client.upsert_entity(
659
+ entity,
660
+ event.port_app_config.get_port_request_options(),
661
+ user_agent_type,
662
+ should_raise=False,
663
+ )
591
664
 
592
665
  except OceanAbortException as ocean_abort:
593
- logger.info(f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count() )
594
- if isinstance(ocean_abort.__cause__,CycleError):
666
+ logger.info(
667
+ f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",
668
+ failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count(),
669
+ )
670
+ if isinstance(ocean_abort.__cause__, CycleError):
595
671
  for entity in event.entity_topological_sorter.get_entities(False):
596
- await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
672
+ await self.entities_state_applier.context.port_client.upsert_entity(
673
+ entity,
674
+ event.port_app_config.get_port_request_options(),
675
+ user_agent_type,
676
+ should_raise=False,
677
+ )
597
678
 
598
- def process_resource_in_subprocess(self,
679
+ def process_resource_in_subprocess(
680
+ self,
599
681
  file_ipc_map: dict[str, FileIPC],
600
682
  resource: ResourceConfig,
601
683
  index: int,
602
684
  user_agent_type: UserAgentType,
603
685
  ) -> None:
604
- logger.info(f"process started successfully for {resource.kind} with index {index}")
686
+ logger.info(
687
+ f"process started successfully for {resource.kind} with index {index}"
688
+ )
605
689
 
606
690
  clear_http_client_context()
691
+
607
692
  async def process_resource_task() -> None:
608
- result = await self._process_resource(
609
- resource, index, user_agent_type
610
- )
693
+ result = await self._process_resource(resource, index, user_agent_type)
611
694
  file_ipc_map["process_resource"].save(result)
612
695
  file_ipc_map["topological_entities"].save(
613
696
  event.entity_topological_sorter.entities
@@ -616,32 +699,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
616
699
  asyncio.run(process_resource_task())
617
700
  logger.info(f"Process finished for {resource.kind} with index {index}")
618
701
 
619
- async def process_resource(self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType) -> tuple[list[Entity], list[Exception]]:
620
- with logger.contextualize(resource_kind=resource.kind, index=index):
621
- if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
622
- id = uuid.uuid4()
623
- logger.info(f"Starting subprocess with id {id}")
624
- file_ipc_map = {
625
- "process_resource": FileIPC(id, "process_resource",([],[IntegrationSubProcessFailedException(f"Subprocess failed for {resource.kind} with index {index}")])),
626
- "topological_entities": FileIPC(id, "topological_entities",[]),
627
- }
628
- process = ProcessWrapper(target=self.process_resource_in_subprocess, args=(file_ipc_map,resource,index,user_agent_type))
629
- process.start()
630
- await process.join_async()
631
-
632
- event.entity_topological_sorter.entities.extend(file_ipc_map["topological_entities"].load())
633
- return file_ipc_map["process_resource"].load()
634
-
635
- else:
636
- return await self._process_resource(resource,index,user_agent_type)
637
-
638
- async def _process_resource(self,resource: ResourceConfig, index: int, user_agent_type: UserAgentType)-> tuple[list[Entity], list[Exception]]:
702
+ async def _process_resource(
703
+ self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
704
+ ) -> tuple[list[Entity], list[Exception]]:
639
705
  # create resource context per resource kind, so resync method could have access to the resource
640
706
  # config as we might have multiple resources in the same event
641
- async with resource_context(resource,index):
707
+ async with resource_context(resource, index):
642
708
  resource_kind_id = f"{resource.kind}-{index}"
643
709
  ocean.metrics.sync_state = SyncState.SYNCING
644
- await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint)
710
+ await ocean.metrics.report_kind_sync_metrics(
711
+ kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
712
+ )
645
713
 
646
714
  task = asyncio.create_task(
647
715
  self._register_in_batches(resource, user_agent_type)
@@ -652,22 +720,87 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
652
720
  if ocean.metrics.sync_state != SyncState.FAILED:
653
721
  ocean.metrics.sync_state = SyncState.COMPLETED
654
722
 
655
- await ocean.metrics.send_metrics_to_webhook(
656
- kind=resource_kind_id
723
+ await ocean.metrics.send_metrics_to_webhook(kind=resource_kind_id)
724
+ await ocean.metrics.report_kind_sync_metrics(
725
+ kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
657
726
  )
658
- await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint)
659
727
 
660
728
  return kind_results
661
729
 
662
- @TimeMetricWithResourceKind(MetricPhase.RESYNC)
663
- async def resync_reconciliation(
730
+ def resync_reconciliation_in_subprocess(
664
731
  self,
732
+ file_ipc_map: dict[str, FileIPC],
665
733
  creation_results: list[tuple[list[Entity], list[Exception]]],
666
734
  did_fetched_current_state: bool,
667
735
  user_agent_type: UserAgentType,
668
736
  app_config: Any,
669
737
  silent: bool = True,
670
738
  ) -> None:
739
+ logger.info("Resync reconciliation subprocess started successfully")
740
+
741
+ clear_http_client_context()
742
+
743
+ async def resync_reconciliation_task() -> None:
744
+ result = await self._resync_reconciliation(
745
+ creation_results,
746
+ did_fetched_current_state,
747
+ user_agent_type,
748
+ app_config,
749
+ silent,
750
+ )
751
+ file_ipc_map["resync_reconciliation"].save(result)
752
+
753
+ asyncio.run(resync_reconciliation_task())
754
+ logger.info("Resync reconciliation subprocess finished")
755
+
756
+ async def process_resource(
757
+ self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
758
+ ) -> tuple[list[Entity], list[Exception]]:
759
+ with logger.contextualize(resource_kind=resource.kind, index=index):
760
+ if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
761
+ id = uuid.uuid4()
762
+ logger.info(f"Starting subprocess with id {id}")
763
+ file_ipc_map = {
764
+ "process_resource": FileIPC(
765
+ str(id),
766
+ "process_resource",
767
+ (
768
+ [],
769
+ [
770
+ IntegrationSubProcessFailedException(
771
+ f"Subprocess failed for {resource.kind} with index {index}"
772
+ )
773
+ ],
774
+ ),
775
+ ),
776
+ "topological_entities": FileIPC(
777
+ str(id), "topological_entities", []
778
+ ),
779
+ }
780
+ process = ProcessWrapper(
781
+ target=self.process_resource_in_subprocess,
782
+ args=(file_ipc_map, resource, index, user_agent_type),
783
+ )
784
+ process.start()
785
+ await process.join_async()
786
+
787
+ event.entity_topological_sorter.entities.extend(
788
+ file_ipc_map["topological_entities"].load()
789
+ )
790
+ return file_ipc_map["process_resource"].load()
791
+
792
+ else:
793
+ return await self._process_resource(resource, index, user_agent_type)
794
+
795
+ @TimeMetricWithResourceKind(MetricPhase.RESYNC)
796
+ async def _resync_reconciliation(
797
+ self,
798
+ creation_results: list[tuple[list[Entity], list[Exception]]],
799
+ did_fetched_current_state: bool,
800
+ user_agent_type: UserAgentType,
801
+ app_config: Any,
802
+ silent: bool = True,
803
+ ) -> bool:
671
804
  """Handle the reconciliation phase of the resync process.
672
805
 
673
806
  This method handles:
@@ -716,13 +849,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
716
849
  logger.info(
717
850
  f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
718
851
  )
719
- entities_at_port = await ocean.port_client.search_entities(
720
- user_agent_type
721
- )
852
+ entities_at_port = await ocean.port_client.search_entities(user_agent_type)
722
853
 
723
854
  await self.entities_state_applier.delete_diff(
724
855
  {"before": entities_at_port, "after": generated_entities},
725
- user_agent_type, app_config.get_entity_deletion_threshold()
856
+ user_agent_type,
857
+ app_config.get_entity_deletion_threshold(),
726
858
  )
727
859
 
728
860
  logger.info("Resync finished successfully")
@@ -738,6 +870,47 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
738
870
 
739
871
  return True
740
872
 
873
+ async def resync_reconciliation(
874
+ self,
875
+ creation_results: list[tuple[list[Entity], list[Exception]]],
876
+ did_fetched_current_state: bool,
877
+ user_agent_type: UserAgentType,
878
+ app_config: Any,
879
+ silent: bool = True,
880
+ ) -> bool:
881
+ if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
882
+ id = uuid.uuid4()
883
+ logger.info(f"Starting resync reconciliation in subprocess with id {id}")
884
+
885
+ file_ipc_map = {
886
+ "resync_reconciliation": FileIPC(
887
+ str(id), "resync_reconciliation", False
888
+ ),
889
+ }
890
+
891
+ process = ProcessWrapper(
892
+ target=self.resync_reconciliation_in_subprocess,
893
+ args=(
894
+ file_ipc_map,
895
+ creation_results,
896
+ did_fetched_current_state,
897
+ user_agent_type,
898
+ app_config,
899
+ silent,
900
+ ),
901
+ )
902
+ process.start()
903
+ await process.join_async()
904
+
905
+ return file_ipc_map["resync_reconciliation"].load()
906
+ else:
907
+ return await self._resync_reconciliation(
908
+ creation_results,
909
+ did_fetched_current_state,
910
+ user_agent_type,
911
+ app_config,
912
+ silent,
913
+ )
741
914
 
742
915
  @TimeMetric(MetricPhase.RESYNC)
743
916
  async def sync_raw_all(
@@ -773,8 +946,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
773
946
  )
774
947
  logger.info(f"Resync will use the following mappings: {app_config.dict()}")
775
948
 
776
- kinds = [f"{resource.kind}-{index}" for index, resource in enumerate(app_config.resources)]
777
- blueprints = [resource.port.entity.mappings.blueprint for resource in app_config.resources]
949
+ kinds = [
950
+ f"{resource.kind}-{index}"
951
+ for index, resource in enumerate(app_config.resources)
952
+ ]
953
+ blueprints = [
954
+ resource.port.entity.mappings.blueprint
955
+ for resource in app_config.resources
956
+ ]
778
957
  ocean.metrics.initialize_metrics(kinds)
779
958
  await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
780
959
 
@@ -799,13 +978,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
799
978
 
800
979
  creation_results: list[tuple[list[Entity], list[Exception]]] = []
801
980
 
802
- multiprocessing.set_start_method('fork', True)
981
+ multiprocessing.set_start_method("fork", True)
803
982
  try:
804
- for index,resource in enumerate(app_config.resources):
805
- logger.info(f"Starting processing resource {resource.kind} with index {index}")
806
- creation_results.append(await self.process_resource(resource,index,user_agent_type))
983
+ for index, resource in enumerate(app_config.resources):
984
+ logger.info(
985
+ f"Starting processing resource {resource.kind} with index {index}"
986
+ )
987
+ creation_results.append(
988
+ await self.process_resource(resource, index, user_agent_type)
989
+ )
807
990
  except asyncio.CancelledError as e:
808
- logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
991
+ logger.warning(
992
+ "Resync aborted successfully, skipping delete phase. This leads to an incomplete state"
993
+ )
809
994
  raise
810
995
  else:
811
996
  success = await self.resync_reconciliation(
@@ -813,11 +998,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
813
998
  did_fetched_current_state,
814
999
  user_agent_type,
815
1000
  app_config,
816
- silent
1001
+ silent,
1002
+ )
1003
+ await ocean.metrics.report_sync_metrics(
1004
+ kinds=[MetricResourceKind.RECONCILIATION]
817
1005
  )
818
- await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
819
1006
  return success
820
1007
  finally:
821
1008
  await ocean.app.cache_provider.clear()
822
- if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
1009
+ if (
1010
+ ocean.app.process_execution_mode
1011
+ == ProcessExecutionMode.multi_process
1012
+ ):
823
1013
  ocean.metrics.cleanup_prometheus_metrics()