port-ocean 0.27.1__tar.gz → 0.27.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {port_ocean-0.27.1 → port_ocean-0.27.10}/PKG-INFO +5 -2
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.Deb +4 -1
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.local +3 -1
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/authentication.py +2 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/client.py +5 -2
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/integrations.py +1 -1
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/settings.py +12 -1
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/kafka.py +14 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +12 -9
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/models.py +1 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/resync_state_updater/updater.py +2 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/sync_raw.py +22 -4
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/utils.py +26 -4
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/async_client.py +7 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/metric/metric.py +13 -11
- port_ocean-0.27.10/port_ocean/helpers/stream.py +71 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/handlers.py +3 -4
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/ocean.py +11 -12
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/conftest.py +7 -1
- port_ocean-0.27.10/port_ocean/tests/core/event_listener/test_kafka.py +70 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +1 -1
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/mixins/test_live_events.py +23 -13
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +32 -27
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/port_client.py +1 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/pyproject.toml +6 -2
- {port_ocean-0.27.1 → port_ocean-0.27.10}/LICENSE.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/README.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.alpine +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.base.builder +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.base.runner +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.dockerignore +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/Makefile +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/README.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/entry_local.sh +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/grpcio.sh +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/integrations/_infra/init.sh +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/bootstrap.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/disk.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/errors.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cache/memory.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cli.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/__init___.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/clean.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/dock.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/defaults/group.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/list_integrations.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/main.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/new.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/pull.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/sail.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/commands/version.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/extensions.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/cli/utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/auth_client.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/auth/oauth_client.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/blueprints.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/entities.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/migrations.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/organization.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/retry_transport.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/types.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/clients/port/utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/config/dynamic.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/consumers/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/consumers/kafka_consumer.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/event.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/metric_resource.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/ocean.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/context/resource.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/clean.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/common.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/defaults/initialize.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/factory.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/http.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/once.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/polling.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/event_listener/webhooks_only.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/api.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/group_queue.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/local_queue.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/events.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/handler.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/live_events.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/integrations/mixins/sync.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/models.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/ocean_types.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/core/utils/utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/debug_cli.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/api.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/clients.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/context.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/core.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/port_defaults.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/exceptions/webhook_processor.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/metric/utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/helpers/retry.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/logger_setup.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/log/sensetive.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/middlewares.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/py.typed +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/run.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/sonar-project.properties +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/test_disk_cache.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/cache/test_memory_cache.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/oauth/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_integrations.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/config/test_config.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/conftest.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/defaults/test_common.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/queue/test_group_queue.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/test_utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/fake_port_api.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/fixtures.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/integration.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/ocean_app.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/helpers/smoke_test.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/log/test_handlers.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_metric.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_ocean.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/test_smoke.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/utils/test_async_iterators.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/tests/utils/test_cache.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/__init__.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/async_http.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/async_iterators.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/cache.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/ipc.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/misc.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/queue_utils.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/repeat.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/signal.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/utils/time.py +0 -0
- {port_ocean-0.27.1 → port_ocean-0.27.10}/port_ocean/version.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: port-ocean
|
3
|
-
Version: 0.27.
|
3
|
+
Version: 0.27.10
|
4
4
|
Summary: Port Ocean is a CLI tool for managing your Port projects.
|
5
5
|
Home-page: https://app.getport.io
|
6
6
|
Keywords: ocean,port-ocean,port
|
@@ -22,12 +22,15 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
22
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
23
23
|
Classifier: Topic :: Utilities
|
24
24
|
Provides-Extra: cli
|
25
|
+
Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
|
25
26
|
Requires-Dist: aiostream (>=0.5.2,<0.7.0)
|
26
27
|
Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
|
27
28
|
Requires-Dist: confluent-kafka (>=2.10.1,<3.0.0)
|
28
29
|
Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
|
29
|
-
Requires-Dist:
|
30
|
+
Requires-Dist: cryptography (>=44.0.1,<45.0.0)
|
31
|
+
Requires-Dist: fastapi (>=0.116.0,<0.117.0)
|
30
32
|
Requires-Dist: httpx (>=0.28.1,<0.29.0)
|
33
|
+
Requires-Dist: ijson (>=3.4.0,<4.0.0)
|
31
34
|
Requires-Dist: jinja2 (>=3.1.6)
|
32
35
|
Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
|
33
36
|
Requires-Dist: jq (>=1.8.0,<2.0.0)
|
@@ -28,11 +28,14 @@ ARG INTEGRATION_VERSION
|
|
28
28
|
ARG BUILD_CONTEXT
|
29
29
|
ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
|
30
30
|
ARG OAUTH_CONFIG_DIR=/app/.config
|
31
|
+
ARG STREAMING_LOCATION=/tmp/ocean/streaming
|
31
32
|
|
32
33
|
ENV LIBRDKAFKA_VERSION=2.8.2 \
|
33
|
-
PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
|
34
|
+
PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR} \
|
35
|
+
STREAMING_LOCATION=${STREAMING_LOCATION}
|
34
36
|
|
35
37
|
RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
|
38
|
+
RUN mkdir -p ${STREAMING_LOCATION}
|
36
39
|
RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
|
37
40
|
|
38
41
|
RUN mkdir -p ${OAUTH_CONFIG_DIR}
|
@@ -33,13 +33,15 @@ RUN apt-get update \
|
|
33
33
|
|
34
34
|
ARG BUILD_CONTEXT
|
35
35
|
ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
|
36
|
+
ARG STREAMING_LOCATION=/tmp/ocean/streaming
|
36
37
|
|
37
38
|
ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
|
38
|
-
|
39
|
+
ENV STREAMING_LOCATION=${STREAMING_LOCATION}
|
39
40
|
# Create /tmp/ocean directory and set permissions
|
40
41
|
|
41
42
|
|
42
43
|
RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
|
44
|
+
RUN mkdir -p ${STREAMING_LOCATION}
|
43
45
|
|
44
46
|
WORKDIR /app
|
45
47
|
|
@@ -35,6 +35,7 @@ class PortAuthentication:
|
|
35
35
|
integration_identifier: str,
|
36
36
|
integration_type: str,
|
37
37
|
integration_version: str,
|
38
|
+
ingest_url: str,
|
38
39
|
):
|
39
40
|
self.client = client
|
40
41
|
self.api_url = api_url
|
@@ -43,6 +44,7 @@ class PortAuthentication:
|
|
43
44
|
self.integration_identifier = integration_identifier
|
44
45
|
self.integration_type = integration_type
|
45
46
|
self.integration_version = integration_version
|
47
|
+
self.ingest_url = ingest_url
|
46
48
|
self.last_token_object: TokenResponse | None = None
|
47
49
|
|
48
50
|
async def _get_token(self, client_id: str, client_secret: str) -> TokenResponse:
|
@@ -1,3 +1,5 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
1
3
|
from loguru import logger
|
2
4
|
|
3
5
|
from port_ocean.clients.port.authentication import PortAuthentication
|
@@ -10,11 +12,10 @@ from port_ocean.clients.port.types import (
|
|
10
12
|
KafkaCreds,
|
11
13
|
)
|
12
14
|
from port_ocean.clients.port.utils import (
|
13
|
-
handle_port_status_code,
|
14
15
|
get_internal_http_client,
|
16
|
+
handle_port_status_code,
|
15
17
|
)
|
16
18
|
from port_ocean.exceptions.clients import KafkaCredentialsNotFound
|
17
|
-
from typing import Any
|
18
19
|
|
19
20
|
|
20
21
|
class PortClient(
|
@@ -32,6 +33,7 @@ class PortClient(
|
|
32
33
|
integration_identifier: str,
|
33
34
|
integration_type: str,
|
34
35
|
integration_version: str,
|
36
|
+
ingest_url: str,
|
35
37
|
):
|
36
38
|
self.api_url = f"{base_url}/v1"
|
37
39
|
self.client = get_internal_http_client(self)
|
@@ -43,6 +45,7 @@ class PortClient(
|
|
43
45
|
integration_identifier,
|
44
46
|
integration_type,
|
45
47
|
integration_version,
|
48
|
+
ingest_url,
|
46
49
|
)
|
47
50
|
EntityClientMixin.__init__(self, self.auth, self.client)
|
48
51
|
IntegrationClientMixin.__init__(
|
@@ -296,7 +296,7 @@ class IntegrationClientMixin:
|
|
296
296
|
logger.debug("starting POST raw data request", raw_data=raw_data)
|
297
297
|
headers = await self.auth.headers()
|
298
298
|
response = await self.client.post(
|
299
|
-
f"{self.auth.
|
299
|
+
f"{self.auth.ingest_url}/lakehouse/integration-type/{self.auth.integration_type}/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
|
300
300
|
headers=headers,
|
301
301
|
json={
|
302
302
|
"items": raw_data,
|
@@ -9,7 +9,6 @@ from pydantic.main import BaseModel
|
|
9
9
|
|
10
10
|
from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
|
11
11
|
from port_ocean.core.event_listener import EventListenerSettingsType
|
12
|
-
|
13
12
|
from port_ocean.core.models import (
|
14
13
|
CachingStorageMode,
|
15
14
|
CreatePortResourcesOrigin,
|
@@ -47,6 +46,7 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
|
|
47
46
|
client_secret: str = Field(..., sensitive=True)
|
48
47
|
base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
|
49
48
|
port_app_config_cache_ttl: int = 60
|
49
|
+
ingest_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://ingest.getport.io")
|
50
50
|
|
51
51
|
|
52
52
|
class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
|
@@ -73,6 +73,13 @@ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
|
|
73
73
|
webhook_url: str | None = Field(default=None)
|
74
74
|
|
75
75
|
|
76
|
+
class StreamingSettings(BaseOceanModel, extra=Extra.allow):
|
77
|
+
enabled: bool = Field(default=False)
|
78
|
+
max_buffer_size_mb: int = Field(default=1024 * 1024 * 20) # 20 mb
|
79
|
+
chunk_size: int = Field(default=1024 * 64) # 64 kb
|
80
|
+
location: str = Field(default="/tmp/ocean/streaming")
|
81
|
+
|
82
|
+
|
76
83
|
class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
77
84
|
_integration_config_model: BaseModel | None = None
|
78
85
|
|
@@ -111,6 +118,10 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
111
118
|
upsert_entities_batch_max_length: int = 20
|
112
119
|
upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
|
113
120
|
lakehouse_enabled: bool = False
|
121
|
+
yield_items_to_parse: bool = False
|
122
|
+
yield_items_to_parse_batch_size: int = 10
|
123
|
+
|
124
|
+
streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
|
114
125
|
|
115
126
|
@validator("process_execution_mode")
|
116
127
|
def validate_process_execution_mode(
|
@@ -16,6 +16,7 @@ from port_ocean.core.event_listener.base import (
|
|
16
16
|
EventListenerEvents,
|
17
17
|
EventListenerSettings,
|
18
18
|
)
|
19
|
+
from pydantic import validator
|
19
20
|
|
20
21
|
|
21
22
|
class KafkaEventListenerSettings(EventListenerSettings):
|
@@ -46,6 +47,19 @@ class KafkaEventListenerSettings(EventListenerSettings):
|
|
46
47
|
kafka_security_enabled: bool = True
|
47
48
|
consumer_poll_timeout: int = 1
|
48
49
|
|
50
|
+
@validator("brokers")
|
51
|
+
@classmethod
|
52
|
+
def parse_brokers(cls, v: str) -> str:
|
53
|
+
# If it's a JSON array string, parse and join
|
54
|
+
if v.strip().startswith("[") and v.strip().endswith("]"):
|
55
|
+
try:
|
56
|
+
parsed = json.loads(v)
|
57
|
+
if isinstance(parsed, list):
|
58
|
+
return ",".join(parsed)
|
59
|
+
except json.JSONDecodeError:
|
60
|
+
pass
|
61
|
+
return v
|
62
|
+
|
49
63
|
def get_changelog_destination_details(self) -> dict[str, Any]:
|
50
64
|
"""
|
51
65
|
Returns the changelog destination configuration for the Kafka event listener.
|
@@ -242,19 +242,21 @@ class JQEntityProcessor(BaseEntityProcessor):
|
|
242
242
|
data: dict[str, Any],
|
243
243
|
raw_entity_mappings: dict[str, Any],
|
244
244
|
items_to_parse: str | None,
|
245
|
+
items_to_parse_name: str,
|
245
246
|
selector_query: str,
|
246
247
|
parse_all: bool = False,
|
247
248
|
) -> tuple[list[MappedEntity], list[Exception]]:
|
248
249
|
raw_data = [data.copy()]
|
249
|
-
if
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
250
|
+
if not ocean.config.yield_items_to_parse:
|
251
|
+
if items_to_parse:
|
252
|
+
items = await self._search(data, items_to_parse)
|
253
|
+
if not isinstance(items, list):
|
254
|
+
logger.warning(
|
255
|
+
f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
|
256
|
+
f" Skipping..."
|
257
|
+
)
|
258
|
+
return [], []
|
259
|
+
raw_data = [{items_to_parse_name: item, **data} for item in items]
|
258
260
|
|
259
261
|
entities, errors = await gather_and_split_errors_from_results(
|
260
262
|
[
|
@@ -303,6 +305,7 @@ class JQEntityProcessor(BaseEntityProcessor):
|
|
303
305
|
self._calculate_entity,
|
304
306
|
raw_entity_mappings,
|
305
307
|
mapping.port.items_to_parse,
|
308
|
+
mapping.port.items_to_parse_name,
|
306
309
|
mapping.selector.query,
|
307
310
|
parse_all,
|
308
311
|
)
|
@@ -39,6 +39,7 @@ class MappingsConfig(BaseModel):
|
|
39
39
|
class PortResourceConfig(BaseModel):
|
40
40
|
entity: MappingsConfig
|
41
41
|
items_to_parse: str | None = Field(alias="itemsToParse")
|
42
|
+
items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
|
42
43
|
|
43
44
|
|
44
45
|
class Selector(BaseModel):
|
@@ -116,7 +116,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
116
116
|
logger.info(
|
117
117
|
f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
|
118
118
|
)
|
119
|
-
results.append(resync_generator_wrapper(task, resource_config.kind))
|
119
|
+
results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
|
120
120
|
else:
|
121
121
|
logger.info(
|
122
122
|
f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
|
@@ -976,6 +976,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
976
976
|
ocean.metrics.initialize_metrics(kinds)
|
977
977
|
await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
|
978
978
|
|
979
|
+
async with metric_resource_context(MetricResourceKind.RUNTIME):
|
980
|
+
ocean.metrics.sync_state = SyncState.SYNCING
|
981
|
+
await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RUNTIME)
|
982
|
+
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RUNTIME])
|
983
|
+
|
979
984
|
# Clear cache
|
980
985
|
await ocean.app.cache_provider.clear()
|
981
986
|
|
@@ -1010,8 +1015,18 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
1010
1015
|
logger.warning(
|
1011
1016
|
"Resync aborted successfully, skipping delete phase. This leads to an incomplete state"
|
1012
1017
|
)
|
1018
|
+
|
1019
|
+
async with metric_resource_context(MetricResourceKind.RUNTIME):
|
1020
|
+
ocean.metrics.sync_state = SyncState.FAILED
|
1021
|
+
await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RUNTIME)
|
1022
|
+
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RUNTIME])
|
1013
1023
|
raise
|
1014
1024
|
else:
|
1025
|
+
async with metric_resource_context(MetricResourceKind.RECONCILIATION):
|
1026
|
+
ocean.metrics.sync_state = SyncState.SYNCING
|
1027
|
+
await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RECONCILIATION)
|
1028
|
+
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
|
1029
|
+
|
1015
1030
|
success = await self.resync_reconciliation(
|
1016
1031
|
creation_results,
|
1017
1032
|
did_fetched_current_state,
|
@@ -1019,9 +1034,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
1019
1034
|
app_config,
|
1020
1035
|
silent,
|
1021
1036
|
)
|
1022
|
-
|
1023
|
-
|
1024
|
-
|
1037
|
+
|
1038
|
+
async with metric_resource_context(MetricResourceKind.RECONCILIATION):
|
1039
|
+
ocean.metrics.sync_state = SyncState.COMPLETED if success else SyncState.FAILED
|
1040
|
+
await ocean.metrics.send_metrics_to_webhook(kind=MetricResourceKind.RECONCILIATION)
|
1041
|
+
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
|
1042
|
+
|
1025
1043
|
return success
|
1026
1044
|
finally:
|
1027
1045
|
await ocean.app.cache_provider.clear()
|
@@ -1,11 +1,13 @@
|
|
1
1
|
from contextlib import contextmanager
|
2
|
-
from typing import Awaitable, Generator, Callable
|
2
|
+
from typing import Awaitable, Generator, Callable, cast
|
3
3
|
|
4
4
|
from loguru import logger
|
5
5
|
|
6
6
|
import asyncio
|
7
7
|
import multiprocessing
|
8
8
|
|
9
|
+
from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
|
10
|
+
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
9
11
|
from port_ocean.core.ocean_types import (
|
10
12
|
ASYNC_GENERATOR_RESYNC_TYPE,
|
11
13
|
RAW_RESULT,
|
@@ -49,7 +51,7 @@ async def resync_function_wrapper(
|
|
49
51
|
|
50
52
|
|
51
53
|
async def resync_generator_wrapper(
|
52
|
-
fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str
|
54
|
+
fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
|
53
55
|
) -> ASYNC_GENERATOR_RESYNC_TYPE:
|
54
56
|
generator = fn(kind)
|
55
57
|
errors = []
|
@@ -58,7 +60,28 @@ async def resync_generator_wrapper(
|
|
58
60
|
try:
|
59
61
|
with resync_error_handling():
|
60
62
|
result = await anext(generator)
|
61
|
-
|
63
|
+
if not ocean.config.yield_items_to_parse:
|
64
|
+
yield validate_result(result)
|
65
|
+
else:
|
66
|
+
batch_size = ocean.config.yield_items_to_parse_batch_size
|
67
|
+
if items_to_parse:
|
68
|
+
for data in result:
|
69
|
+
items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
|
70
|
+
if not isinstance(items, list):
|
71
|
+
logger.warning(
|
72
|
+
f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
|
73
|
+
f" Skipping..."
|
74
|
+
)
|
75
|
+
yield []
|
76
|
+
raw_data = [{"item": item, **data} for item in items]
|
77
|
+
while True:
|
78
|
+
raw_data_batch = raw_data[:batch_size]
|
79
|
+
yield raw_data_batch
|
80
|
+
raw_data = raw_data[batch_size:]
|
81
|
+
if len(raw_data) == 0:
|
82
|
+
break
|
83
|
+
else:
|
84
|
+
yield validate_result(result)
|
62
85
|
except OceanAbortException as error:
|
63
86
|
errors.append(error)
|
64
87
|
ocean.metrics.inc_metric(
|
@@ -97,7 +120,6 @@ class ProcessWrapper(multiprocessing.Process):
|
|
97
120
|
logger.error(f"Process {self.pid} failed with exit code {self.exitcode}")
|
98
121
|
else:
|
99
122
|
logger.info(f"Process {self.pid} finished with exit code {self.exitcode}")
|
100
|
-
ocean.metrics.cleanup_prometheus_metrics(self.pid)
|
101
123
|
return super().join()
|
102
124
|
|
103
125
|
def clear_http_client_context() -> None:
|
@@ -4,6 +4,7 @@ import httpx
|
|
4
4
|
from loguru import logger
|
5
5
|
|
6
6
|
from port_ocean.helpers.retry import RetryTransport
|
7
|
+
from port_ocean.helpers.stream import Stream
|
7
8
|
|
8
9
|
|
9
10
|
class OceanAsyncClient(httpx.AsyncClient):
|
@@ -50,3 +51,9 @@ class OceanAsyncClient(httpx.AsyncClient):
|
|
50
51
|
logger=logger,
|
51
52
|
**(self._transport_kwargs or {}),
|
52
53
|
)
|
54
|
+
|
55
|
+
async def get_stream(self, url: str, **kwargs: Any) -> Stream:
|
56
|
+
req = self.build_request("GET", url, **kwargs)
|
57
|
+
response = await self.send(req, stream=True)
|
58
|
+
response.raise_for_status()
|
59
|
+
return Stream(response)
|
@@ -1,21 +1,22 @@
|
|
1
1
|
import os
|
2
|
-
from typing import
|
3
|
-
|
4
|
-
from port_ocean.exceptions.context import ResourceContextNotFoundError
|
2
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
3
|
+
|
5
4
|
import prometheus_client
|
6
|
-
from httpx import AsyncClient
|
7
|
-
from fastapi.responses import PlainTextResponse
|
8
|
-
from loguru import logger
|
9
|
-
from port_ocean.context import metric_resource, resource
|
10
|
-
from prometheus_client import Gauge
|
11
5
|
import prometheus_client.openmetrics
|
12
6
|
import prometheus_client.openmetrics.exposition
|
13
7
|
import prometheus_client.parser
|
14
|
-
from
|
8
|
+
from fastapi import APIRouter
|
9
|
+
from fastapi.responses import PlainTextResponse
|
10
|
+
from httpx import AsyncClient
|
11
|
+
from loguru import logger
|
12
|
+
from prometheus_client import Gauge, multiprocess
|
13
|
+
|
14
|
+
from port_ocean.context import metric_resource, resource
|
15
|
+
from port_ocean.exceptions.context import ResourceContextNotFoundError
|
15
16
|
|
16
17
|
if TYPE_CHECKING:
|
17
|
-
from port_ocean.config.settings import MetricsSettings, IntegrationSettings
|
18
18
|
from port_ocean.clients.port.client import PortClient
|
19
|
+
from port_ocean.config.settings import IntegrationSettings, MetricsSettings
|
19
20
|
|
20
21
|
|
21
22
|
class MetricPhase:
|
@@ -61,6 +62,7 @@ class SyncState:
|
|
61
62
|
class MetricResourceKind:
|
62
63
|
RECONCILIATION = "__reconciliation__"
|
63
64
|
RESYNC = "__resync__"
|
65
|
+
RUNTIME = "__runtime__"
|
64
66
|
|
65
67
|
|
66
68
|
# Registry for core and custom metrics
|
@@ -278,7 +280,7 @@ class Metrics:
|
|
278
280
|
try:
|
279
281
|
return metric_resource.metric_resource.metric_resource_kind
|
280
282
|
except ResourceContextNotFoundError:
|
281
|
-
return
|
283
|
+
return MetricResourceKind.RUNTIME
|
282
284
|
|
283
285
|
def generate_latest(self) -> str:
|
284
286
|
return prometheus_client.openmetrics.exposition.generate_latest(
|
@@ -0,0 +1,71 @@
|
|
1
|
+
import os
|
2
|
+
from typing import Any, AsyncGenerator
|
3
|
+
import uuid
|
4
|
+
|
5
|
+
import aiofiles
|
6
|
+
import httpx
|
7
|
+
import ijson # type: ignore[import-untyped]
|
8
|
+
from cryptography.fernet import Fernet
|
9
|
+
|
10
|
+
import port_ocean.context.ocean as ocean_context
|
11
|
+
|
12
|
+
|
13
|
+
class Stream:
|
14
|
+
def __init__(self, response: httpx.Response):
|
15
|
+
self.response = response
|
16
|
+
self.headers = response.headers
|
17
|
+
self.status_code = response.status_code
|
18
|
+
|
19
|
+
async def _byte_stream(
|
20
|
+
self, chunk_size: int | None = None
|
21
|
+
) -> AsyncGenerator[bytes, None]:
|
22
|
+
if chunk_size is None:
|
23
|
+
chunk_size = ocean_context.ocean.config.streaming.chunk_size
|
24
|
+
|
25
|
+
file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
|
26
|
+
|
27
|
+
crypt = Fernet(Fernet.generate_key())
|
28
|
+
|
29
|
+
try:
|
30
|
+
async for chunk in self.response.aiter_bytes(chunk_size=chunk_size):
|
31
|
+
async with aiofiles.open(f"{file_name}", "ab") as f:
|
32
|
+
if len(chunk) > 0:
|
33
|
+
await f.write(crypt.encrypt(chunk))
|
34
|
+
await f.write(b"\n")
|
35
|
+
finally:
|
36
|
+
await self.response.aclose()
|
37
|
+
|
38
|
+
try:
|
39
|
+
async with aiofiles.open(f"{file_name}", mode="rb") as f:
|
40
|
+
while True:
|
41
|
+
line = await f.readline()
|
42
|
+
if not line:
|
43
|
+
break
|
44
|
+
data = crypt.decrypt(line)
|
45
|
+
yield data
|
46
|
+
finally:
|
47
|
+
try:
|
48
|
+
os.remove(file_name)
|
49
|
+
except FileNotFoundError:
|
50
|
+
pass
|
51
|
+
|
52
|
+
async def get_json_stream(
|
53
|
+
self,
|
54
|
+
target_items: str = "",
|
55
|
+
max_buffer_size_mb: int | None = None,
|
56
|
+
) -> AsyncGenerator[list[dict[str, Any]], None]:
|
57
|
+
if max_buffer_size_mb is None:
|
58
|
+
max_buffer_size_mb = ocean_context.ocean.config.streaming.max_buffer_size_mb
|
59
|
+
|
60
|
+
events = ijson.sendable_list()
|
61
|
+
coro = ijson.items_coro(events, target_items)
|
62
|
+
current_buffer_size = 0
|
63
|
+
async for chunk in self._byte_stream():
|
64
|
+
coro.send(chunk)
|
65
|
+
current_buffer_size += len(chunk)
|
66
|
+
if current_buffer_size >= max_buffer_size_mb:
|
67
|
+
if len(events) > 0:
|
68
|
+
yield events
|
69
|
+
events.clear()
|
70
|
+
current_buffer_size = 0
|
71
|
+
yield events
|
@@ -3,16 +3,16 @@ import logging
|
|
3
3
|
import sys
|
4
4
|
import threading
|
5
5
|
import time
|
6
|
+
from copy import deepcopy
|
6
7
|
from datetime import datetime
|
7
8
|
from logging.handlers import MemoryHandler
|
9
|
+
from traceback import format_exception
|
8
10
|
from typing import Any
|
9
11
|
|
10
12
|
from loguru import logger
|
11
13
|
|
12
14
|
from port_ocean import Ocean
|
13
15
|
from port_ocean.context.ocean import ocean
|
14
|
-
from copy import deepcopy
|
15
|
-
from traceback import format_exception
|
16
16
|
|
17
17
|
|
18
18
|
def _serialize_record(record: logging.LogRecord) -> dict[str, Any]:
|
@@ -53,7 +53,6 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
53
53
|
return None
|
54
54
|
|
55
55
|
def emit(self, record: logging.LogRecord) -> None:
|
56
|
-
|
57
56
|
self._serialized_buffer.append(_serialize_record(record))
|
58
57
|
super().emit(record)
|
59
58
|
|
@@ -106,4 +105,4 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
106
105
|
try:
|
107
106
|
await _ocean.port_client.ingest_integration_logs(logs_to_send)
|
108
107
|
except Exception as e:
|
109
|
-
logger.
|
108
|
+
logger.error(f"Failed to send logs to Port with error: {e}")
|
@@ -1,21 +1,18 @@
|
|
1
1
|
import asyncio
|
2
2
|
import sys
|
3
|
-
from contextlib import asynccontextmanager
|
4
3
|
import threading
|
4
|
+
from contextlib import asynccontextmanager
|
5
5
|
from typing import Any, AsyncIterator, Callable, Dict, Type
|
6
6
|
|
7
|
-
from
|
8
|
-
from port_ocean.cache.disk import DiskCacheProvider
|
9
|
-
from port_ocean.cache.memory import InMemoryCacheProvider
|
10
|
-
from port_ocean.core.models import ProcessExecutionMode
|
11
|
-
import port_ocean.helpers.metric.metric
|
12
|
-
|
13
|
-
from fastapi import FastAPI, APIRouter
|
14
|
-
|
7
|
+
from fastapi import APIRouter, FastAPI
|
15
8
|
from loguru import logger
|
16
9
|
from pydantic import BaseModel
|
17
10
|
from starlette.types import Receive, Scope, Send
|
18
11
|
|
12
|
+
import port_ocean.helpers.metric.metric
|
13
|
+
from port_ocean.cache.base import CacheProvider
|
14
|
+
from port_ocean.cache.disk import DiskCacheProvider
|
15
|
+
from port_ocean.cache.memory import InMemoryCacheProvider
|
19
16
|
from port_ocean.clients.port.client import PortClient
|
20
17
|
from port_ocean.config.settings import (
|
21
18
|
IntegrationConfiguration,
|
@@ -26,16 +23,17 @@ from port_ocean.context.ocean import (
|
|
26
23
|
ocean,
|
27
24
|
)
|
28
25
|
from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
|
26
|
+
from port_ocean.core.handlers.webhook.processor_manager import (
|
27
|
+
LiveEventsProcessorManager,
|
28
|
+
)
|
29
29
|
from port_ocean.core.integrations.base import BaseIntegration
|
30
|
+
from port_ocean.core.models import ProcessExecutionMode
|
30
31
|
from port_ocean.log.sensetive import sensitive_log_filter
|
31
32
|
from port_ocean.middlewares import request_handler
|
32
33
|
from port_ocean.utils.misc import IntegrationStateStatus
|
33
34
|
from port_ocean.utils.repeat import repeat_every
|
34
35
|
from port_ocean.utils.signal import signal_handler
|
35
36
|
from port_ocean.version import __integration_version__
|
36
|
-
from port_ocean.core.handlers.webhook.processor_manager import (
|
37
|
-
LiveEventsProcessorManager,
|
38
|
-
)
|
39
37
|
|
40
38
|
|
41
39
|
class Ocean:
|
@@ -69,6 +67,7 @@ class Ocean:
|
|
69
67
|
integration_identifier=self.config.integration.identifier,
|
70
68
|
integration_type=self.config.integration.type,
|
71
69
|
integration_version=__integration_version__,
|
70
|
+
ingest_url=self.config.port.ingest_url,
|
72
71
|
)
|
73
72
|
self.cache_provider: CacheProvider = self._get_caching_provider()
|
74
73
|
self.process_execution_mode: ProcessExecutionMode = (
|
@@ -96,7 +96,13 @@ def mock_http_client() -> MagicMock:
|
|
96
96
|
@pytest.fixture
|
97
97
|
def mock_port_client(mock_http_client: MagicMock) -> PortClient:
|
98
98
|
mock_port_client = PortClient(
|
99
|
-
MagicMock(),
|
99
|
+
MagicMock(),
|
100
|
+
MagicMock(),
|
101
|
+
MagicMock(),
|
102
|
+
MagicMock(),
|
103
|
+
MagicMock(),
|
104
|
+
MagicMock(),
|
105
|
+
MagicMock(),
|
100
106
|
)
|
101
107
|
mock_port_client.auth = AsyncMock()
|
102
108
|
mock_port_client.auth.headers = AsyncMock(
|