port-ocean 0.27.1__tar.gz → 0.27.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. {port_ocean-0.27.1 → port_ocean-0.27.10}/PKG-INFO +5 -2
  2. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.Deb +4 -1
  3. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.local +3 -1
  4. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/authentication.py +2 -0
  5. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/client.py +5 -2
  6. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/integrations.py +1 -1
  7. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/settings.py +12 -1
  8. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/kafka.py +14 -0
  9. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +12 -9
  10. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/models.py +1 -0
  11. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/resync_state_updater/updater.py +2 -0
  12. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/sync_raw.py +22 -4
  13. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/utils.py +26 -4
  14. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/async_client.py +7 -0
  15. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/metric/metric.py +13 -11
  16. port_ocean-0.27.10/port_ocean/helpers/stream.py +71 -0
  17. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/handlers.py +3 -4
  18. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/ocean.py +11 -12
  19. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/conftest.py +7 -1
  20. port_ocean-0.27.10/port_ocean/tests/core/event_listener/test_kafka.py +70 -0
  21. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +1 -1
  22. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/mixins/test_live_events.py +23 -13
  23. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +32 -27
  24. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/port_client.py +1 -0
  25. {port_ocean-0.27.1 → port_ocean-0.27.10}/pyproject.toml +6 -2
  26. {port_ocean-0.27.1 → port_ocean-0.27.10}/LICENSE.md +0 -0
  27. {port_ocean-0.27.1 → port_ocean-0.27.10}/README.md +0 -0
  28. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.alpine +0 -0
  29. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.base.builder +0 -0
  30. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.base.runner +0 -0
  31. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.dockerignore +0 -0
  32. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Makefile +0 -0
  33. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/README.md +0 -0
  34. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/entry_local.sh +0 -0
  35. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/grpcio.sh +0 -0
  36. {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/init.sh +0 -0
  37. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/__init__.py +0 -0
  38. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/bootstrap.py +0 -0
  39. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/__init__.py +0 -0
  40. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/base.py +0 -0
  41. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/disk.py +0 -0
  42. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/errors.py +0 -0
  43. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/memory.py +0 -0
  44. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/__init__.py +0 -0
  45. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cli.py +0 -0
  46. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/__init__.py +0 -0
  47. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/__init___.py +0 -0
  48. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/clean.py +0 -0
  49. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/dock.py +0 -0
  50. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/group.py +0 -0
  51. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/list_integrations.py +0 -0
  52. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/main.py +0 -0
  53. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/new.py +0 -0
  54. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/pull.py +0 -0
  55. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/sail.py +0 -0
  56. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/version.py +0 -0
  57. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/__init__.py +0 -0
  58. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
  59. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/extensions.py +0 -0
  60. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
  61. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
  62. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
  63. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
  64. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
  65. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
  66. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
  67. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
  68. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
  69. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
  70. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
  71. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
  72. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
  73. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
  74. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
  75. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
  76. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py +0 -0
  77. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
  78. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/utils.py +0 -0
  79. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/__init__.py +0 -0
  80. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/__init__.py +0 -0
  81. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/auth_client.py +0 -0
  82. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/oauth_client.py +0 -0
  83. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/__init__.py +0 -0
  84. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/__init__.py +0 -0
  85. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/blueprints.py +0 -0
  86. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/entities.py +0 -0
  87. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/migrations.py +0 -0
  88. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/organization.py +0 -0
  89. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/retry_transport.py +0 -0
  90. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/types.py +0 -0
  91. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/utils.py +0 -0
  92. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/__init__.py +0 -0
  93. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/base.py +0 -0
  94. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/dynamic.py +0 -0
  95. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/consumers/__init__.py +0 -0
  96. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/consumers/kafka_consumer.py +0 -0
  97. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/__init__.py +0 -0
  98. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/event.py +0 -0
  99. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/metric_resource.py +0 -0
  100. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/ocean.py +0 -0
  101. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/resource.py +0 -0
  102. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/__init__.py +0 -0
  103. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/__init__.py +0 -0
  104. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/clean.py +0 -0
  105. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/common.py +0 -0
  106. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/initialize.py +0 -0
  107. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/__init__.py +0 -0
  108. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/base.py +0 -0
  109. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/factory.py +0 -0
  110. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/http.py +0 -0
  111. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/once.py +0 -0
  112. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/polling.py +0 -0
  113. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/webhooks_only.py +0 -0
  114. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/__init__.py +0 -0
  115. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/base.py +0 -0
  116. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
  117. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
  118. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/__init__.py +0 -0
  119. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
  120. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
  121. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
  122. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
  123. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/base.py +0 -0
  124. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
  125. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/api.py +0 -0
  126. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/base.py +0 -0
  127. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/__init__.py +0 -0
  128. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
  129. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/group_queue.py +0 -0
  130. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/local_queue.py +0 -0
  131. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
  132. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/__init__.py +0 -0
  133. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
  134. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
  135. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
  136. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/__init__.py +0 -0
  137. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/base.py +0 -0
  138. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/__init__.py +0 -0
  139. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/events.py +0 -0
  140. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/handler.py +0 -0
  141. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/live_events.py +0 -0
  142. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/sync.py +0 -0
  143. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/models.py +0 -0
  144. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/ocean_types.py +0 -0
  145. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
  146. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/utils/utils.py +0 -0
  147. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/debug_cli.py +0 -0
  148. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/__init__.py +0 -0
  149. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/api.py +0 -0
  150. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/base.py +0 -0
  151. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/clients.py +0 -0
  152. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/context.py +0 -0
  153. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/core.py +0 -0
  154. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/port_defaults.py +0 -0
  155. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/utils.py +0 -0
  156. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/webhook_processor.py +0 -0
  157. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/__init__.py +0 -0
  158. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/metric/utils.py +0 -0
  159. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/retry.py +0 -0
  160. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/__init__.py +0 -0
  161. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/logger_setup.py +0 -0
  162. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/sensetive.py +0 -0
  163. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/middlewares.py +0 -0
  164. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/py.typed +0 -0
  165. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/run.py +0 -0
  166. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/sonar-project.properties +0 -0
  167. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/__init__.py +0 -0
  168. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/__init__.py +0 -0
  169. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/test_disk_cache.py +0 -0
  170. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/test_memory_cache.py +0 -0
  171. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/__init__.py +0 -0
  172. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/oauth/__init__.py +0 -0
  173. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
  174. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
  175. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_integrations.py +0 -0
  176. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
  177. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/config/test_config.py +0 -0
  178. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/conftest.py +0 -0
  179. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/defaults/test_common.py +0 -0
  180. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
  181. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +0 -0
  182. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
  183. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
  184. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/queue/test_group_queue.py +0 -0
  185. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
  186. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
  187. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
  188. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/test_utils.py +0 -0
  189. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
  190. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
  191. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/__init__.py +0 -0
  192. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/fake_port_api.py +0 -0
  193. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/fixtures.py +0 -0
  194. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/integration.py +0 -0
  195. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/ocean_app.py +0 -0
  196. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/smoke_test.py +0 -0
  197. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/log/test_handlers.py +0 -0
  198. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_metric.py +0 -0
  199. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_ocean.py +0 -0
  200. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_smoke.py +0 -0
  201. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/utils/test_async_iterators.py +0 -0
  202. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/utils/test_cache.py +0 -0
  203. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/__init__.py +0 -0
  204. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/async_http.py +0 -0
  205. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/async_iterators.py +0 -0
  206. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/cache.py +0 -0
  207. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/ipc.py +0 -0
  208. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/misc.py +0 -0
  209. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/queue_utils.py +0 -0
  210. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/repeat.py +0 -0
  211. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/signal.py +0 -0
  212. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/time.py +0 -0
  213. {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: port-ocean
3
- Version: 0.27.1
3
+ Version: 0.27.10
4
4
  Summary: Port Ocean is a CLI tool for managing your Port projects.
5
5
  Home-page: https://app.getport.io
6
6
  Keywords: ocean,port-ocean,port
@@ -22,12 +22,15 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
22
22
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
23
  Classifier: Topic :: Utilities
24
24
  Provides-Extra: cli
25
+ Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
25
26
  Requires-Dist: aiostream (>=0.5.2,<0.7.0)
26
27
  Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
27
28
  Requires-Dist: confluent-kafka (>=2.10.1,<3.0.0)
28
29
  Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
29
- Requires-Dist: fastapi (>=0.115.3,<0.116.0)
30
+ Requires-Dist: cryptography (>=44.0.1,<45.0.0)
31
+ Requires-Dist: fastapi (>=0.116.0,<0.117.0)
30
32
  Requires-Dist: httpx (>=0.28.1,<0.29.0)
33
+ Requires-Dist: ijson (>=3.4.0,<4.0.0)
31
34
  Requires-Dist: jinja2 (>=3.1.6)
32
35
  Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
33
36
  Requires-Dist: jq (>=1.8.0,<2.0.0)
@@ -28,11 +28,14 @@ ARG INTEGRATION_VERSION
28
28
  ARG BUILD_CONTEXT
29
29
  ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
30
30
  ARG OAUTH_CONFIG_DIR=/app/.config
31
+ ARG STREAMING_LOCATION=/tmp/ocean/streaming
31
32
 
32
33
  ENV LIBRDKAFKA_VERSION=2.8.2 \
33
- PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
34
+ PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR} \
35
+ STREAMING_LOCATION=${STREAMING_LOCATION}
34
36
 
35
37
  RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
38
+ RUN mkdir -p ${STREAMING_LOCATION}
36
39
  RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
37
40
 
38
41
  RUN mkdir -p ${OAUTH_CONFIG_DIR}
@@ -33,13 +33,15 @@ RUN apt-get update \
33
33
 
34
34
  ARG BUILD_CONTEXT
35
35
  ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
36
+ ARG STREAMING_LOCATION=/tmp/ocean/streaming
36
37
 
37
38
  ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
38
-
39
+ ENV STREAMING_LOCATION=${STREAMING_LOCATION}
39
40
  # Create /tmp/ocean directory and set permissions
40
41
 
41
42
 
42
43
  RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
44
+ RUN mkdir -p ${STREAMING_LOCATION}
43
45
 
44
46
  WORKDIR /app
45
47
 
@@ -35,6 +35,7 @@ class PortAuthentication:
35
35
  integration_identifier: str,
36
36
  integration_type: str,
37
37
  integration_version: str,
38
+ ingest_url: str,
38
39
  ):
39
40
  self.client = client
40
41
  self.api_url = api_url
@@ -43,6 +44,7 @@ class PortAuthentication:
43
44
  self.integration_identifier = integration_identifier
44
45
  self.integration_type = integration_type
45
46
  self.integration_version = integration_version
47
+ self.ingest_url = ingest_url
46
48
  self.last_token_object: TokenResponse | None = None
47
49
 
48
50
  async def _get_token(self, client_id: str, client_secret: str) -> TokenResponse:
@@ -1,3 +1,5 @@
1
+ from typing import Any
2
+
1
3
  from loguru import logger
2
4
 
3
5
  from port_ocean.clients.port.authentication import PortAuthentication
@@ -10,11 +12,10 @@ from port_ocean.clients.port.types import (
10
12
  KafkaCreds,
11
13
  )
12
14
  from port_ocean.clients.port.utils import (
13
- handle_port_status_code,
14
15
  get_internal_http_client,
16
+ handle_port_status_code,
15
17
  )
16
18
  from port_ocean.exceptions.clients import KafkaCredentialsNotFound
17
- from typing import Any
18
19
 
19
20
 
20
21
  class PortClient(
@@ -32,6 +33,7 @@ class PortClient(
32
33
  integration_identifier: str,
33
34
  integration_type: str,
34
35
  integration_version: str,
36
+ ingest_url: str,
35
37
  ):
36
38
  self.api_url = f"{base_url}/v1"
37
39
  self.client = get_internal_http_client(self)
@@ -43,6 +45,7 @@ class PortClient(
43
45
  integration_identifier,
44
46
  integration_type,
45
47
  integration_version,
48
+ ingest_url,
46
49
  )
47
50
  EntityClientMixin.__init__(self, self.auth, self.client)
48
51
  IntegrationClientMixin.__init__(
@@ -296,7 +296,7 @@ class IntegrationClientMixin:
296
296
  logger.debug("starting POST raw data request", raw_data=raw_data)
297
297
  headers = await self.auth.headers()
298
298
  response = await self.client.post(
299
- f"{self.auth.api_url}/lakehouse/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
299
+ f"{self.auth.ingest_url}/lakehouse/integration-type/{self.auth.integration_type}/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
300
300
  headers=headers,
301
301
  json={
302
302
  "items": raw_data,
@@ -9,7 +9,6 @@ from pydantic.main import BaseModel
9
9
 
10
10
  from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
11
11
  from port_ocean.core.event_listener import EventListenerSettingsType
12
-
13
12
  from port_ocean.core.models import (
14
13
  CachingStorageMode,
15
14
  CreatePortResourcesOrigin,
@@ -47,6 +46,7 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
47
46
  client_secret: str = Field(..., sensitive=True)
48
47
  base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
49
48
  port_app_config_cache_ttl: int = 60
49
+ ingest_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://ingest.getport.io")
50
50
 
51
51
 
52
52
  class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
@@ -73,6 +73,13 @@ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
73
73
  webhook_url: str | None = Field(default=None)
74
74
 
75
75
 
76
+ class StreamingSettings(BaseOceanModel, extra=Extra.allow):
77
+ enabled: bool = Field(default=False)
78
+ max_buffer_size_mb: int = Field(default=1024 * 1024 * 20) # 20 mb
79
+ chunk_size: int = Field(default=1024 * 64) # 64 kb
80
+ location: str = Field(default="/tmp/ocean/streaming")
81
+
82
+
76
83
  class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
77
84
  _integration_config_model: BaseModel | None = None
78
85
 
@@ -111,6 +118,10 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
111
118
  upsert_entities_batch_max_length: int = 20
112
119
  upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
113
120
  lakehouse_enabled: bool = False
121
+ yield_items_to_parse: bool = False
122
+ yield_items_to_parse_batch_size: int = 10
123
+
124
+ streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
114
125
 
115
126
  @validator("process_execution_mode")
116
127
  def validate_process_execution_mode(
@@ -16,6 +16,7 @@ from port_ocean.core.event_listener.base import (
16
16
  EventListenerEvents,
17
17
  EventListenerSettings,
18
18
  )
19
+ from pydantic import validator
19
20
 
20
21
 
21
22
  class KafkaEventListenerSettings(EventListenerSettings):
@@ -46,6 +47,19 @@ class KafkaEventListenerSettings(EventListenerSettings):
46
47
  kafka_security_enabled: bool = True
47
48
  consumer_poll_timeout: int = 1
48
49
 
50
+ @validator("brokers")
51
+ @classmethod
52
+ def parse_brokers(cls, v: str) -> str:
53
+ # If it's a JSON array string, parse and join
54
+ if v.strip().startswith("[") and v.strip().endswith("]"):
55
+ try:
56
+ parsed = json.loads(v)
57
+ if isinstance(parsed, list):
58
+ return ",".join(parsed)
59
+ except json.JSONDecodeError:
60
+ pass
61
+ return v
62
+
49
63
  def get_changelog_destination_details(self) -> dict[str, Any]:
50
64
  """
51
65
  Returns the changelog destination configuration for the Kafka event listener.
@@ -242,19 +242,21 @@ class JQEntityProcessor(BaseEntityProcessor):
242
242
  data: dict[str, Any],
243
243
  raw_entity_mappings: dict[str, Any],
244
244
  items_to_parse: str | None,
245
+ items_to_parse_name: str,
245
246
  selector_query: str,
246
247
  parse_all: bool = False,
247
248
  ) -> tuple[list[MappedEntity], list[Exception]]:
248
249
  raw_data = [data.copy()]
249
- if items_to_parse:
250
- items = await self._search(data, items_to_parse)
251
- if not isinstance(items, list):
252
- logger.warning(
253
- f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
254
- f" Skipping..."
255
- )
256
- return [], []
257
- raw_data = [{"item": item, **data} for item in items]
250
+ if not ocean.config.yield_items_to_parse:
251
+ if items_to_parse:
252
+ items = await self._search(data, items_to_parse)
253
+ if not isinstance(items, list):
254
+ logger.warning(
255
+ f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
256
+ f" Skipping..."
257
+ )
258
+ return [], []
259
+ raw_data = [{items_to_parse_name: item, **data} for item in items]
258
260
 
259
261
  entities, errors = await gather_and_split_errors_from_results(
260
262
  [
@@ -303,6 +305,7 @@ class JQEntityProcessor(BaseEntityProcessor):
303
305
  self._calculate_entity,
304
306
  raw_entity_mappings,
305
307
  mapping.port.items_to_parse,
308
+ mapping.port.items_to_parse_name,
306
309
  mapping.selector.query,
307
310
  parse_all,
308
311
  )
@@ -39,6 +39,7 @@ class MappingsConfig(BaseModel):
39
39
  class PortResourceConfig(BaseModel):
40
40
  entity: MappingsConfig
41
41
  items_to_parse: str | None = Field(alias="itemsToParse")
42
+ items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
42
43
 
43
44
 
44
45
  class Selector(BaseModel):
@@ -91,6 +91,8 @@ class ResyncStateUpdater:
91
91
  value=int(status == IntegrationStateStatus.Completed),
92
92
  )
93
93
 
94
+ ocean.metrics.sync_state = status.value
95
+
94
96
  await ocean.metrics.send_metrics_to_webhook(
95
97
  kind=ocean.metrics.current_resource_kind()
96
98
  )
@@ -116,7 +116,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
116
116
  logger.info(
117
117
  f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
118
118
  )
119
- results.append(resync_generator_wrapper(task, resource_config.kind))
119
+ results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
120
120
  else:
121
121
  logger.info(
122
122
  f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
@@ -976,6 +976,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
976
976
  ocean.metrics.initialize_metrics(kinds)
977
977
  await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
978
978
 
979
+ async with metric_resource_context(MetricResourceKind.RUNTIME):
980
+ ocean.metrics.sync_state = SyncState.SYNCING
981
+ await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RUNTIME)
982
+ await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RUNTIME])
983
+
979
984
  # Clear cache
980
985
  await ocean.app.cache_provider.clear()
981
986
 
@@ -1010,8 +1015,18 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
1010
1015
  logger.warning(
1011
1016
  "Resync aborted successfully, skipping delete phase. This leads to an incomplete state"
1012
1017
  )
1018
+
1019
+ async with metric_resource_context(MetricResourceKind.RUNTIME):
1020
+ ocean.metrics.sync_state = SyncState.FAILED
1021
+ await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RUNTIME)
1022
+ await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RUNTIME])
1013
1023
  raise
1014
1024
  else:
1025
+ async with metric_resource_context(MetricResourceKind.RECONCILIATION):
1026
+ ocean.metrics.sync_state = SyncState.SYNCING
1027
+ await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RECONCILIATION)
1028
+ await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
1029
+
1015
1030
  success = await self.resync_reconciliation(
1016
1031
  creation_results,
1017
1032
  did_fetched_current_state,
@@ -1019,9 +1034,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
1019
1034
  app_config,
1020
1035
  silent,
1021
1036
  )
1022
- await ocean.metrics.report_sync_metrics(
1023
- kinds=[MetricResourceKind.RECONCILIATION]
1024
- )
1037
+
1038
+ async with metric_resource_context(MetricResourceKind.RECONCILIATION):
1039
+ ocean.metrics.sync_state = SyncState.COMPLETED if success else SyncState.FAILED
1040
+ await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RECONCILIATION)
1041
+ await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
1042
+
1025
1043
  return success
1026
1044
  finally:
1027
1045
  await ocean.app.cache_provider.clear()
@@ -1,11 +1,13 @@
1
1
  from contextlib import contextmanager
2
- from typing import Awaitable, Generator, Callable
2
+ from typing import Awaitable, Generator, Callable, cast
3
3
 
4
4
  from loguru import logger
5
5
 
6
6
  import asyncio
7
7
  import multiprocessing
8
8
 
9
+ from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
10
+ from port_ocean.core.handlers.port_app_config.models import ResourceConfig
9
11
  from port_ocean.core.ocean_types import (
10
12
  ASYNC_GENERATOR_RESYNC_TYPE,
11
13
  RAW_RESULT,
@@ -49,7 +51,7 @@ async def resync_function_wrapper(
49
51
 
50
52
 
51
53
  async def resync_generator_wrapper(
52
- fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str
54
+ fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
53
55
  ) -> ASYNC_GENERATOR_RESYNC_TYPE:
54
56
  generator = fn(kind)
55
57
  errors = []
@@ -58,7 +60,28 @@ async def resync_generator_wrapper(
58
60
  try:
59
61
  with resync_error_handling():
60
62
  result = await anext(generator)
61
- yield validate_result(result)
63
+ if not ocean.config.yield_items_to_parse:
64
+ yield validate_result(result)
65
+ else:
66
+ batch_size = ocean.config.yield_items_to_parse_batch_size
67
+ if items_to_parse:
68
+ for data in result:
69
+ items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
70
+ if not isinstance(items, list):
71
+ logger.warning(
72
+ f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
73
+ f" Skipping..."
74
+ )
75
+ yield []
76
+ raw_data = [{"item": item, **data} for item in items]
77
+ while True:
78
+ raw_data_batch = raw_data[:batch_size]
79
+ yield raw_data_batch
80
+ raw_data = raw_data[batch_size:]
81
+ if len(raw_data) == 0:
82
+ break
83
+ else:
84
+ yield validate_result(result)
62
85
  except OceanAbortException as error:
63
86
  errors.append(error)
64
87
  ocean.metrics.inc_metric(
@@ -97,7 +120,6 @@ class ProcessWrapper(multiprocessing.Process):
97
120
  logger.error(f"Process {self.pid} failed with exit code {self.exitcode}")
98
121
  else:
99
122
  logger.info(f"Process {self.pid} finished with exit code {self.exitcode}")
100
- ocean.metrics.cleanup_prometheus_metrics(self.pid)
101
123
  return super().join()
102
124
 
103
125
  def clear_http_client_context() -> None:
@@ -4,6 +4,7 @@ import httpx
4
4
  from loguru import logger
5
5
 
6
6
  from port_ocean.helpers.retry import RetryTransport
7
+ from port_ocean.helpers.stream import Stream
7
8
 
8
9
 
9
10
  class OceanAsyncClient(httpx.AsyncClient):
@@ -50,3 +51,9 @@ class OceanAsyncClient(httpx.AsyncClient):
50
51
  logger=logger,
51
52
  **(self._transport_kwargs or {}),
52
53
  )
54
+
55
+ async def get_stream(self, url: str, **kwargs: Any) -> Stream:
56
+ req = self.build_request("GET", url, **kwargs)
57
+ response = await self.send(req, stream=True)
58
+ response.raise_for_status()
59
+ return Stream(response)
@@ -1,21 +1,22 @@
1
1
  import os
2
- from typing import Any, TYPE_CHECKING, Optional, Dict, List, Tuple
3
- from fastapi import APIRouter
4
- from port_ocean.exceptions.context import ResourceContextNotFoundError
2
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
3
+
5
4
  import prometheus_client
6
- from httpx import AsyncClient
7
- from fastapi.responses import PlainTextResponse
8
- from loguru import logger
9
- from port_ocean.context import metric_resource, resource
10
- from prometheus_client import Gauge
11
5
  import prometheus_client.openmetrics
12
6
  import prometheus_client.openmetrics.exposition
13
7
  import prometheus_client.parser
14
- from prometheus_client import multiprocess
8
+ from fastapi import APIRouter
9
+ from fastapi.responses import PlainTextResponse
10
+ from httpx import AsyncClient
11
+ from loguru import logger
12
+ from prometheus_client import Gauge, multiprocess
13
+
14
+ from port_ocean.context import metric_resource, resource
15
+ from port_ocean.exceptions.context import ResourceContextNotFoundError
15
16
 
16
17
  if TYPE_CHECKING:
17
- from port_ocean.config.settings import MetricsSettings, IntegrationSettings
18
18
  from port_ocean.clients.port.client import PortClient
19
+ from port_ocean.config.settings import IntegrationSettings, MetricsSettings
19
20
 
20
21
 
21
22
  class MetricPhase:
@@ -61,6 +62,7 @@ class SyncState:
61
62
  class MetricResourceKind:
62
63
  RECONCILIATION = "__reconciliation__"
63
64
  RESYNC = "__resync__"
65
+ RUNTIME = "__runtime__"
64
66
 
65
67
 
66
68
  # Registry for core and custom metrics
@@ -278,7 +280,7 @@ class Metrics:
278
280
  try:
279
281
  return metric_resource.metric_resource.metric_resource_kind
280
282
  except ResourceContextNotFoundError:
281
- return "__runtime__"
283
+ return MetricResourceKind.RUNTIME
282
284
 
283
285
  def generate_latest(self) -> str:
284
286
  return prometheus_client.openmetrics.exposition.generate_latest(
@@ -0,0 +1,71 @@
1
+ import os
2
+ from typing import Any, AsyncGenerator
3
+ import uuid
4
+
5
+ import aiofiles
6
+ import httpx
7
+ import ijson # type: ignore[import-untyped]
8
+ from cryptography.fernet import Fernet
9
+
10
+ import port_ocean.context.ocean as ocean_context
11
+
12
+
13
+ class Stream:
14
+ def __init__(self, response: httpx.Response):
15
+ self.response = response
16
+ self.headers = response.headers
17
+ self.status_code = response.status_code
18
+
19
+ async def _byte_stream(
20
+ self, chunk_size: int | None = None
21
+ ) -> AsyncGenerator[bytes, None]:
22
+ if chunk_size is None:
23
+ chunk_size = ocean_context.ocean.config.streaming.chunk_size
24
+
25
+ file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
26
+
27
+ crypt = Fernet(Fernet.generate_key())
28
+
29
+ try:
30
+ async for chunk in self.response.aiter_bytes(chunk_size=chunk_size):
31
+ async with aiofiles.open(f"{file_name}", "ab") as f:
32
+ if len(chunk) > 0:
33
+ await f.write(crypt.encrypt(chunk))
34
+ await f.write(b"\n")
35
+ finally:
36
+ await self.response.aclose()
37
+
38
+ try:
39
+ async with aiofiles.open(f"{file_name}", mode="rb") as f:
40
+ while True:
41
+ line = await f.readline()
42
+ if not line:
43
+ break
44
+ data = crypt.decrypt(line)
45
+ yield data
46
+ finally:
47
+ try:
48
+ os.remove(file_name)
49
+ except FileNotFoundError:
50
+ pass
51
+
52
+ async def get_json_stream(
53
+ self,
54
+ target_items: str = "",
55
+ max_buffer_size_mb: int | None = None,
56
+ ) -> AsyncGenerator[list[dict[str, Any]], None]:
57
+ if max_buffer_size_mb is None:
58
+ max_buffer_size_mb = ocean_context.ocean.config.streaming.max_buffer_size_mb
59
+
60
+ events = ijson.sendable_list()
61
+ coro = ijson.items_coro(events, target_items)
62
+ current_buffer_size = 0
63
+ async for chunk in self._byte_stream():
64
+ coro.send(chunk)
65
+ current_buffer_size += len(chunk)
66
+ if current_buffer_size >= max_buffer_size_mb:
67
+ if len(events) > 0:
68
+ yield events
69
+ events.clear()
70
+ current_buffer_size = 0
71
+ yield events
@@ -3,16 +3,16 @@ import logging
3
3
  import sys
4
4
  import threading
5
5
  import time
6
+ from copy import deepcopy
6
7
  from datetime import datetime
7
8
  from logging.handlers import MemoryHandler
9
+ from traceback import format_exception
8
10
  from typing import Any
9
11
 
10
12
  from loguru import logger
11
13
 
12
14
  from port_ocean import Ocean
13
15
  from port_ocean.context.ocean import ocean
14
- from copy import deepcopy
15
- from traceback import format_exception
16
16
 
17
17
 
18
18
  def _serialize_record(record: logging.LogRecord) -> dict[str, Any]:
@@ -53,7 +53,6 @@ class HTTPMemoryHandler(MemoryHandler):
53
53
  return None
54
54
 
55
55
  def emit(self, record: logging.LogRecord) -> None:
56
-
57
56
  self._serialized_buffer.append(_serialize_record(record))
58
57
  super().emit(record)
59
58
 
@@ -106,4 +105,4 @@ class HTTPMemoryHandler(MemoryHandler):
106
105
  try:
107
106
  await _ocean.port_client.ingest_integration_logs(logs_to_send)
108
107
  except Exception as e:
109
- logger.debug(f"Failed to send logs to Port with error: {e}")
108
+ logger.error(f"Failed to send logs to Port with error: {e}")
@@ -1,21 +1,18 @@
1
1
  import asyncio
2
2
  import sys
3
- from contextlib import asynccontextmanager
4
3
  import threading
4
+ from contextlib import asynccontextmanager
5
5
  from typing import Any, AsyncIterator, Callable, Dict, Type
6
6
 
7
- from port_ocean.cache.base import CacheProvider
8
- from port_ocean.cache.disk import DiskCacheProvider
9
- from port_ocean.cache.memory import InMemoryCacheProvider
10
- from port_ocean.core.models import ProcessExecutionMode
11
- import port_ocean.helpers.metric.metric
12
-
13
- from fastapi import FastAPI, APIRouter
14
-
7
+ from fastapi import APIRouter, FastAPI
15
8
  from loguru import logger
16
9
  from pydantic import BaseModel
17
10
  from starlette.types import Receive, Scope, Send
18
11
 
12
+ import port_ocean.helpers.metric.metric
13
+ from port_ocean.cache.base import CacheProvider
14
+ from port_ocean.cache.disk import DiskCacheProvider
15
+ from port_ocean.cache.memory import InMemoryCacheProvider
19
16
  from port_ocean.clients.port.client import PortClient
20
17
  from port_ocean.config.settings import (
21
18
  IntegrationConfiguration,
@@ -26,16 +23,17 @@ from port_ocean.context.ocean import (
26
23
  ocean,
27
24
  )
28
25
  from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
26
+ from port_ocean.core.handlers.webhook.processor_manager import (
27
+ LiveEventsProcessorManager,
28
+ )
29
29
  from port_ocean.core.integrations.base import BaseIntegration
30
+ from port_ocean.core.models import ProcessExecutionMode
30
31
  from port_ocean.log.sensetive import sensitive_log_filter
31
32
  from port_ocean.middlewares import request_handler
32
33
  from port_ocean.utils.misc import IntegrationStateStatus
33
34
  from port_ocean.utils.repeat import repeat_every
34
35
  from port_ocean.utils.signal import signal_handler
35
36
  from port_ocean.version import __integration_version__
36
- from port_ocean.core.handlers.webhook.processor_manager import (
37
- LiveEventsProcessorManager,
38
- )
39
37
 
40
38
 
41
39
  class Ocean:
@@ -69,6 +67,7 @@ class Ocean:
69
67
  integration_identifier=self.config.integration.identifier,
70
68
  integration_type=self.config.integration.type,
71
69
  integration_version=__integration_version__,
70
+ ingest_url=self.config.port.ingest_url,
72
71
  )
73
72
  self.cache_provider: CacheProvider = self._get_caching_provider()
74
73
  self.process_execution_mode: ProcessExecutionMode = (
@@ -96,7 +96,13 @@ def mock_http_client() -> MagicMock:
96
96
  @pytest.fixture
97
97
  def mock_port_client(mock_http_client: MagicMock) -> PortClient:
98
98
  mock_port_client = PortClient(
99
- MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
99
+ MagicMock(),
100
+ MagicMock(),
101
+ MagicMock(),
102
+ MagicMock(),
103
+ MagicMock(),
104
+ MagicMock(),
105
+ MagicMock(),
100
106
  )
101
107
  mock_port_client.auth = AsyncMock()
102
108
  mock_port_client.auth.headers = AsyncMock(