port-ocean 0.22.12__tar.gz → 0.23.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- {port_ocean-0.22.12 → port_ocean-0.23.0}/PKG-INFO +1 -1
- port_ocean-0.23.0/integrations/_infra/Dockerfile.local +41 -0
- port_ocean-0.23.0/integrations/_infra/entry_local.sh +27 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/__init__.py +6 -1
- port_ocean-0.23.0/port_ocean/cache/base.py +25 -0
- port_ocean-0.23.0/port_ocean/cache/disk.py +61 -0
- port_ocean-0.23.0/port_ocean/cache/errors.py +10 -0
- port_ocean-0.23.0/port_ocean/cache/memory.py +36 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/settings.py +9 -2
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/sync_raw.py +80 -27
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/models.py +10 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/metric/metric.py +4 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/ocean.py +28 -1
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/run.py +0 -1
- port_ocean-0.23.0/port_ocean/tests/cache/__init__.py +1 -0
- port_ocean-0.23.0/port_ocean/tests/cache/test_disk_cache.py +92 -0
- port_ocean-0.23.0/port_ocean/tests/cache/test_memory_cache.py +59 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/conftest.py +13 -3
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +2 -28
- port_ocean-0.23.0/port_ocean/tests/helpers/__init__.py +0 -0
- port_ocean-0.23.0/port_ocean/tests/utils/test_cache.py +274 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/cache.py +35 -12
- port_ocean-0.23.0/port_ocean/utils/ipc.py +30 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/pyproject.toml +4 -1
- port_ocean-0.22.12/port_ocean/tests/utils/test_cache.py +0 -189
- {port_ocean-0.22.12 → port_ocean-0.23.0}/LICENSE.md +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/README.md +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.Deb +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.alpine +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.base.builder +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.base.runner +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Dockerfile.dockerignore +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/Makefile +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/grpcio.sh +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/integrations/_infra/init.sh +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/bootstrap.py +0 -0
- {port_ocean-0.22.12/port_ocean/cli/cookiecutter → port_ocean-0.23.0/port_ocean/cache}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cli.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/__init___.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/clean.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/dock.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/defaults/group.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/list_integrations.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/main.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/new.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/pull.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/sail.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/commands/version.py +0 -0
- {port_ocean-0.22.12/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests → port_ocean-0.23.0/port_ocean/cli/cookiecutter}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/extensions.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
- {port_ocean-0.22.12/port_ocean/clients → port_ocean-0.23.0/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/cli/utils.py +0 -0
- {port_ocean-0.22.12/port_ocean/clients/auth → port_ocean-0.23.0/port_ocean/clients}/__init__.py +0 -0
- {port_ocean-0.22.12/port_ocean/clients/port → port_ocean-0.23.0/port_ocean/clients/auth}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/auth/auth_client.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/auth/oauth_client.py +0 -0
- {port_ocean-0.22.12/port_ocean/clients/port/mixins → port_ocean-0.23.0/port_ocean/clients/port}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/authentication.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/client.py +0 -0
- {port_ocean-0.22.12/port_ocean/config → port_ocean-0.23.0/port_ocean/clients/port/mixins}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/blueprints.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/entities.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/integrations.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/migrations.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/mixins/organization.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/retry_transport.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/types.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/clients/port/utils.py +0 -0
- {port_ocean-0.22.12/port_ocean/consumers → port_ocean-0.23.0/port_ocean/config}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/config/dynamic.py +0 -0
- {port_ocean-0.22.12/port_ocean/context → port_ocean-0.23.0/port_ocean/consumers}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/consumers/kafka_consumer.py +0 -0
- {port_ocean-0.22.12/port_ocean/core → port_ocean-0.23.0/port_ocean/context}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/event.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/ocean.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/context/resource.py +0 -0
- {port_ocean-0.22.12/port_ocean/core/handlers/entities_state_applier/port → port_ocean-0.23.0/port_ocean/core}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/clean.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/common.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/defaults/initialize.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/factory.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/http.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/kafka.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/once.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/polling.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/event_listener/webhooks_only.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
- {port_ocean-0.22.12/port_ocean/core/handlers/webhook → port_ocean-0.23.0/port_ocean/core/handlers/entities_state_applier/port}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/api.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/port_app_config/models.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/queue/local_queue.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/resync_state_updater/updater.py +0 -0
- {port_ocean-0.22.12/port_ocean/core/integrations → port_ocean-0.23.0/port_ocean/core/handlers/webhook}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
- {port_ocean-0.22.12/port_ocean/exceptions → port_ocean-0.23.0/port_ocean/core/integrations}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/events.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/handler.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/live_events.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/sync.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/integrations/mixins/utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/ocean_types.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/core/utils/utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/debug_cli.py +0 -0
- {port_ocean-0.22.12/port_ocean/helpers → port_ocean-0.23.0/port_ocean/exceptions}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/api.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/clients.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/context.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/core.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/port_defaults.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/exceptions/webhook_processor.py +0 -0
- {port_ocean-0.22.12/port_ocean/log → port_ocean-0.23.0/port_ocean/helpers}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/async_client.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/metric/utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/helpers/retry.py +0 -0
- {port_ocean-0.22.12/port_ocean/tests → port_ocean-0.23.0/port_ocean/log}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/handlers.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/logger_setup.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/log/sensetive.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/middlewares.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/py.typed +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/sonar-project.properties +0 -0
- {port_ocean-0.22.12/port_ocean/tests/clients → port_ocean-0.23.0/port_ocean/tests}/__init__.py +0 -0
- {port_ocean-0.22.12/port_ocean/tests/clients/oauth → port_ocean-0.23.0/port_ocean/tests/clients}/__init__.py +0 -0
- {port_ocean-0.22.12/port_ocean/tests/helpers → port_ocean-0.23.0/port_ocean/tests/clients/oauth}/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/conftest.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/defaults/test_common.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/mixins/test_live_events.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/test_utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/fake_port_api.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/fixtures.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/integration.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/ocean_app.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/port_client.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/helpers/smoke_test.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/log/test_handlers.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_metric.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_ocean.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/test_smoke.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/tests/utils/test_async_iterators.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/__init__.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/async_http.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/async_iterators.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/misc.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/queue_utils.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/repeat.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/signal.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/utils/time.py +0 -0
- {port_ocean-0.22.12 → port_ocean-0.23.0}/port_ocean/version.py +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
ARG BASE_PYTHON_IMAGE=debian:trixie-slim
|
|
2
|
+
# debian:trixie-slim - Python 3.12
|
|
3
|
+
FROM ${BASE_PYTHON_IMAGE}
|
|
4
|
+
|
|
5
|
+
RUN apt-get update \
|
|
6
|
+
&& apt-get install -y --no-install-recommends librdkafka-dev python3 \
|
|
7
|
+
&& apt-get clean
|
|
8
|
+
RUN apt-get update \
|
|
9
|
+
&& apt-get install -y \
|
|
10
|
+
--no-install-recommends \
|
|
11
|
+
wget \
|
|
12
|
+
g++ \
|
|
13
|
+
libssl-dev \
|
|
14
|
+
autoconf \
|
|
15
|
+
automake \
|
|
16
|
+
libtool \
|
|
17
|
+
curl \
|
|
18
|
+
librdkafka-dev \
|
|
19
|
+
python3 \
|
|
20
|
+
python3-pip \
|
|
21
|
+
python3-poetry \
|
|
22
|
+
build-essential\
|
|
23
|
+
git \
|
|
24
|
+
python3-venv \
|
|
25
|
+
&& apt-get clean
|
|
26
|
+
|
|
27
|
+
ARG BUILD_CONTEXT
|
|
28
|
+
|
|
29
|
+
WORKDIR /app
|
|
30
|
+
|
|
31
|
+
COPY . .
|
|
32
|
+
RUN rm -rf .venv-docker ${BUILD_CONTEXT}/.venv-docker
|
|
33
|
+
RUN python3 -m venv .venv-docker
|
|
34
|
+
RUN python3 -m venv ${BUILD_CONTEXT}/.venv-docker
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
WORKDIR /app/${BUILD_CONTEXT}
|
|
38
|
+
|
|
39
|
+
WORKDIR /app
|
|
40
|
+
|
|
41
|
+
ENTRYPOINT ["./integrations/_infra/entry_local.sh"]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
mkdir -p /tmp/prometheus_multiproc_dir
|
|
3
|
+
export PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc_dir
|
|
4
|
+
if [ -z "$BUILD_CONTEXT" ]; then
|
|
5
|
+
echo "BUILD_CONTEXT is not set"
|
|
6
|
+
exit 1
|
|
7
|
+
fi
|
|
8
|
+
|
|
9
|
+
if [ ! -d ".venv-docker" ]; then
|
|
10
|
+
/usr/bin/python3 -m venv .venv-docker
|
|
11
|
+
source .venv-docker/bin/activate
|
|
12
|
+
python -m pip install poetry
|
|
13
|
+
python -m poetry install
|
|
14
|
+
fi
|
|
15
|
+
|
|
16
|
+
cd $BUILD_CONTEXT
|
|
17
|
+
|
|
18
|
+
if [ ! -d ".venv-docker" ]; then
|
|
19
|
+
/usr/bin/python3 -m venv .venv-docker
|
|
20
|
+
source .venv-docker/bin/activate
|
|
21
|
+
python -m pip install poetry
|
|
22
|
+
python -m poetry install
|
|
23
|
+
fi
|
|
24
|
+
source .venv-docker/bin/activate
|
|
25
|
+
python -m pip install -e ../../
|
|
26
|
+
|
|
27
|
+
ocean sail
|
|
@@ -8,4 +8,9 @@ from .run import run # noqa: E402
|
|
|
8
8
|
from .version import __integration_version__, __version__ # noqa: E402
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
__all__ = [
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Ocean",
|
|
13
|
+
"run",
|
|
14
|
+
"__version__",
|
|
15
|
+
"__integration_version__",
|
|
16
|
+
]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any, Optional
|
|
3
|
+
|
|
4
|
+
from port_ocean.core.models import CachingStorageMode
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CacheProvider(ABC):
|
|
8
|
+
"""Base class for cache providers that defines the contract for all cache implementations."""
|
|
9
|
+
|
|
10
|
+
STORAGE_TYPE: CachingStorageMode
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
async def get(self, key: str) -> Optional[Any]:
|
|
14
|
+
"""Get a value from the cache."""
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
async def set(self, key: str, value: Any) -> None:
|
|
19
|
+
"""Set a value in the cache."""
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
async def clear(self) -> None:
|
|
24
|
+
"""Clear all values from the cache."""
|
|
25
|
+
pass
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, Optional
|
|
4
|
+
|
|
5
|
+
from port_ocean.cache.base import CacheProvider
|
|
6
|
+
from port_ocean.cache.errors import FailedToReadCacheError, FailedToWriteCacheError
|
|
7
|
+
from port_ocean.core.models import CachingStorageMode
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class FailedToReadCacheFileError(FailedToReadCacheError):
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FailedToWriteCacheFileError(FailedToWriteCacheError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DiskCacheProvider(CacheProvider):
|
|
19
|
+
STORAGE_TYPE = CachingStorageMode.disk
|
|
20
|
+
|
|
21
|
+
def __init__(self, cache_dir: str | None = None) -> None:
|
|
22
|
+
if cache_dir is None:
|
|
23
|
+
cache_dir = ".ocean_cache"
|
|
24
|
+
self._cache_dir = Path(cache_dir)
|
|
25
|
+
self._cache_dir.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
|
|
27
|
+
def _get_cache_path(self, key: str) -> Path:
|
|
28
|
+
return self._cache_dir / f"{key}.pkl"
|
|
29
|
+
|
|
30
|
+
async def get(self, key: str) -> Optional[Any]:
|
|
31
|
+
cache_path = self._get_cache_path(key)
|
|
32
|
+
if not cache_path.exists():
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
with open(cache_path, "rb") as f:
|
|
37
|
+
return pickle.load(f)
|
|
38
|
+
except (pickle.PickleError, EOFError) as e:
|
|
39
|
+
raise FailedToReadCacheFileError(
|
|
40
|
+
f"Failed to read cache file: {cache_path}: {str(e)}"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
async def set(self, key: str, value: Any) -> None:
|
|
44
|
+
cache_path = self._get_cache_path(key)
|
|
45
|
+
try:
|
|
46
|
+
with open(cache_path, "wb") as f:
|
|
47
|
+
pickle.dump(value, f)
|
|
48
|
+
except (pickle.PickleError, IOError) as e:
|
|
49
|
+
raise FailedToWriteCacheFileError(
|
|
50
|
+
f"Failed to write cache file: {cache_path}: {str(e)}"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
async def clear(self) -> None:
|
|
54
|
+
try:
|
|
55
|
+
for cache_file in self._cache_dir.glob("*.pkl"):
|
|
56
|
+
try:
|
|
57
|
+
cache_file.unlink()
|
|
58
|
+
except OSError:
|
|
59
|
+
pass
|
|
60
|
+
except OSError:
|
|
61
|
+
pass
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from typing import Any, Optional
|
|
2
|
+
from port_ocean.cache.base import CacheProvider
|
|
3
|
+
from port_ocean.cache.errors import FailedToReadCacheError, FailedToWriteCacheError
|
|
4
|
+
from port_ocean.core.models import CachingStorageMode
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FailedToReadCacheMemoryError(FailedToReadCacheError):
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FailedToWriteCacheMemoryError(FailedToWriteCacheError):
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class InMemoryCacheProvider(CacheProvider):
|
|
16
|
+
CACHE_KEY = "cache"
|
|
17
|
+
STORAGE_TYPE = CachingStorageMode.memory
|
|
18
|
+
|
|
19
|
+
def __init__(self, caching_storage: dict[str, Any] | None = None) -> None:
|
|
20
|
+
self._storage = caching_storage or {}
|
|
21
|
+
self._storage[self.CACHE_KEY] = self._storage.get(self.CACHE_KEY, {})
|
|
22
|
+
|
|
23
|
+
async def get(self, key: str) -> Optional[Any]:
|
|
24
|
+
try:
|
|
25
|
+
return self._storage.get(self.CACHE_KEY, {}).get(key)
|
|
26
|
+
except KeyError as e:
|
|
27
|
+
raise FailedToReadCacheMemoryError(f"Failed to read cache: {str(e)}")
|
|
28
|
+
|
|
29
|
+
async def set(self, key: str, value: Any) -> None:
|
|
30
|
+
try:
|
|
31
|
+
self._storage[self.CACHE_KEY][key] = value
|
|
32
|
+
except KeyError as e:
|
|
33
|
+
raise FailedToWriteCacheMemoryError(f"Failed to write cache: {str(e)}")
|
|
34
|
+
|
|
35
|
+
async def clear(self) -> None:
|
|
36
|
+
self._storage[self.CACHE_KEY].clear()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, Literal, Type, cast
|
|
1
|
+
from typing import Any, Literal, Optional, Type, cast
|
|
2
2
|
|
|
3
3
|
from pydantic import AnyHttpUrl, Extra, parse_obj_as, parse_raw_as
|
|
4
4
|
from pydantic.class_validators import root_validator, validator
|
|
@@ -8,7 +8,12 @@ from pydantic.main import BaseModel
|
|
|
8
8
|
|
|
9
9
|
from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
|
|
10
10
|
from port_ocean.core.event_listener import EventListenerSettingsType
|
|
11
|
-
from port_ocean.core.models import
|
|
11
|
+
from port_ocean.core.models import (
|
|
12
|
+
CachingStorageMode,
|
|
13
|
+
CreatePortResourcesOrigin,
|
|
14
|
+
Runtime,
|
|
15
|
+
ProcessExecutionMode,
|
|
16
|
+
)
|
|
12
17
|
from port_ocean.utils.misc import get_integration_name, get_spec_file
|
|
13
18
|
|
|
14
19
|
LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
|
|
@@ -93,6 +98,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
|
93
98
|
)
|
|
94
99
|
max_event_processing_seconds: float = 90.0
|
|
95
100
|
max_wait_seconds_before_shutdown: float = 5.0
|
|
101
|
+
caching_storage_mode: Optional[CachingStorageMode] = Field(default=None)
|
|
102
|
+
process_execution_mode: Optional[ProcessExecutionMode] = Field(default=None)
|
|
96
103
|
|
|
97
104
|
@validator("metrics", pre=True)
|
|
98
105
|
def validate_metrics(cls, v: Any) -> MetricsSettings | dict[str, Any] | None:
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import uuid
|
|
2
3
|
from graphlib import CycleError
|
|
3
4
|
import inspect
|
|
4
5
|
import typing
|
|
5
6
|
from typing import Callable, Awaitable, Any
|
|
6
|
-
|
|
7
|
+
import multiprocessing
|
|
7
8
|
import httpx
|
|
8
9
|
from loguru import logger
|
|
9
|
-
|
|
10
10
|
from port_ocean.clients.port.types import UserAgentType
|
|
11
11
|
from port_ocean.context.event import TriggerType, event_context, EventType, event
|
|
12
12
|
from port_ocean.context.ocean import ocean
|
|
@@ -20,7 +20,7 @@ from port_ocean.core.integrations.mixins.utils import (
|
|
|
20
20
|
resync_generator_wrapper,
|
|
21
21
|
resync_function_wrapper,
|
|
22
22
|
)
|
|
23
|
-
from port_ocean.core.models import Entity
|
|
23
|
+
from port_ocean.core.models import Entity, ProcessExecutionMode
|
|
24
24
|
from port_ocean.core.ocean_types import (
|
|
25
25
|
RAW_RESULT,
|
|
26
26
|
RESYNC_RESULT,
|
|
@@ -33,6 +33,7 @@ from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gath
|
|
|
33
33
|
from port_ocean.exceptions.core import OceanAbortException
|
|
34
34
|
from port_ocean.helpers.metric.metric import SyncState, MetricType, MetricPhase
|
|
35
35
|
from port_ocean.helpers.metric.utils import TimeMetric
|
|
36
|
+
from port_ocean.utils.ipc import FileIPC
|
|
36
37
|
|
|
37
38
|
SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
|
|
38
39
|
|
|
@@ -267,7 +268,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
267
268
|
objects_diff[0].entity_selector_diff.passed, user_agent_type
|
|
268
269
|
)
|
|
269
270
|
|
|
270
|
-
|
|
271
271
|
return CalculationResult(
|
|
272
272
|
number_of_transformed_entities=len(objects_diff[0].entity_selector_diff.passed),
|
|
273
273
|
entity_selector_diff=objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
|
|
@@ -344,8 +344,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
344
344
|
user_agent_type,
|
|
345
345
|
send_raw_data_examples_amount=send_raw_data_examples_amount
|
|
346
346
|
)
|
|
347
|
-
errors.extend(calculation_result.errors)
|
|
348
347
|
passed_entities.extend(calculation_result.entity_selector_diff.passed)
|
|
348
|
+
errors.extend(calculation_result.errors)
|
|
349
349
|
number_of_transformed_entities += calculation_result.number_of_transformed_entities
|
|
350
350
|
except* OceanAbortException as error:
|
|
351
351
|
ocean.metrics.sync_state = SyncState.FAILED
|
|
@@ -355,6 +355,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
355
355
|
f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
|
|
356
356
|
)
|
|
357
357
|
|
|
358
|
+
|
|
358
359
|
ocean.metrics.set_metric(
|
|
359
360
|
name=MetricType.SUCCESS_NAME,
|
|
360
361
|
labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
|
|
@@ -584,6 +585,72 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
584
585
|
for entity in event.entity_topological_sorter.get_entities(False):
|
|
585
586
|
await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
|
|
586
587
|
|
|
588
|
+
def process_resource_in_subprocess(self,
|
|
589
|
+
file_ipc_map: dict[str, FileIPC],
|
|
590
|
+
resource: ResourceConfig,
|
|
591
|
+
index: int,
|
|
592
|
+
user_agent_type: UserAgentType,
|
|
593
|
+
) -> None:
|
|
594
|
+
logger.info(f"process started successfully for {resource.kind} with index {index}")
|
|
595
|
+
|
|
596
|
+
async def process_resource_task() -> None:
|
|
597
|
+
result = await self._process_resource(
|
|
598
|
+
resource, index, user_agent_type
|
|
599
|
+
)
|
|
600
|
+
file_ipc_map["process_resource"].save(result)
|
|
601
|
+
file_ipc_map["topological_entities"].save(
|
|
602
|
+
event.entity_topological_sorter.entities
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
asyncio.run(process_resource_task())
|
|
606
|
+
logger.info(f"Process finished for {resource.kind} with index {index}")
|
|
607
|
+
|
|
608
|
+
async def process_resource(self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType) -> tuple[list[Entity], list[Exception]]:
|
|
609
|
+
if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
|
|
610
|
+
id = uuid.uuid4()
|
|
611
|
+
logger.info(f"Starting subprocess with id {id}")
|
|
612
|
+
file_ipc_map = {
|
|
613
|
+
"process_resource": FileIPC(id, "process_resource",([],[])),
|
|
614
|
+
"topological_entities": FileIPC(id, "topological_entities",[]),
|
|
615
|
+
}
|
|
616
|
+
process = multiprocessing.Process(target=self.process_resource_in_subprocess, args=(file_ipc_map,resource,index,user_agent_type))
|
|
617
|
+
process.start()
|
|
618
|
+
process.join()
|
|
619
|
+
if process.exitcode != 0:
|
|
620
|
+
logger.error(f"Process {id} failed with exit code {process.exitcode}")
|
|
621
|
+
event.entity_topological_sorter.entities.extend(file_ipc_map["topological_entities"].load())
|
|
622
|
+
return file_ipc_map["process_resource"].load()
|
|
623
|
+
|
|
624
|
+
else:
|
|
625
|
+
return await self._process_resource(resource,index,user_agent_type)
|
|
626
|
+
|
|
627
|
+
async def _process_resource(self,resource: ResourceConfig, index: int, user_agent_type: UserAgentType)-> tuple[list[Entity], list[Exception]]:
|
|
628
|
+
# create resource context per resource kind, so resync method could have access to the resource
|
|
629
|
+
# config as we might have multiple resources in the same event
|
|
630
|
+
async with resource_context(resource,index):
|
|
631
|
+
resource_kind_id = f"{resource.kind}-{index}"
|
|
632
|
+
ocean.metrics.sync_state = SyncState.SYNCING
|
|
633
|
+
task = asyncio.create_task(
|
|
634
|
+
self._register_in_batches(resource, user_agent_type)
|
|
635
|
+
)
|
|
636
|
+
event.on_abort(lambda: task.cancel())
|
|
637
|
+
kind_results: tuple[list[Entity], list[Exception]] = await task
|
|
638
|
+
ocean.metrics.set_metric(
|
|
639
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
640
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
|
|
641
|
+
value=len(kind_results[0])
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
if ocean.metrics.sync_state != SyncState.FAILED:
|
|
645
|
+
ocean.metrics.sync_state = SyncState.COMPLETED
|
|
646
|
+
|
|
647
|
+
await ocean.metrics.send_metrics_to_webhook(
|
|
648
|
+
kind=resource_kind_id
|
|
649
|
+
)
|
|
650
|
+
# await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
|
|
651
|
+
|
|
652
|
+
return kind_results
|
|
653
|
+
|
|
587
654
|
@TimeMetric(MetricPhase.RESYNC)
|
|
588
655
|
async def sync_raw_all(
|
|
589
656
|
self,
|
|
@@ -622,6 +689,9 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
622
689
|
ocean.metrics.initialize_metrics(kinds)
|
|
623
690
|
# await ocean.metrics.report_sync_metrics(kinds=kinds) # TODO: uncomment this when end points are ready
|
|
624
691
|
|
|
692
|
+
# Clear cache
|
|
693
|
+
await ocean.app.cache_provider.clear()
|
|
694
|
+
|
|
625
695
|
# Execute resync_start hooks
|
|
626
696
|
for resync_start_fn in self.event_strategy["resync_start"]:
|
|
627
697
|
await resync_start_fn()
|
|
@@ -640,32 +710,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
640
710
|
|
|
641
711
|
creation_results: list[tuple[list[Entity], list[Exception]]] = []
|
|
642
712
|
|
|
643
|
-
|
|
713
|
+
multiprocessing.set_start_method('fork', True)
|
|
644
714
|
try:
|
|
645
715
|
for index,resource in enumerate(app_config.resources):
|
|
646
|
-
# create resource context per resource kind, so resync method could have access to the resource
|
|
647
|
-
# config as we might have multiple resources in the same event
|
|
648
|
-
async with resource_context(resource,index):
|
|
649
|
-
resource_kind_id = f"{resource.kind}-{index}"
|
|
650
|
-
ocean.metrics.sync_state = SyncState.SYNCING
|
|
651
|
-
# await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
|
|
652
|
-
|
|
653
|
-
task = asyncio.create_task(
|
|
654
|
-
self._register_in_batches(resource, user_agent_type)
|
|
655
|
-
)
|
|
656
716
|
|
|
657
|
-
|
|
658
|
-
kind_results: tuple[list[Entity], list[Exception]] = await task
|
|
717
|
+
logger.info(f"Starting processing resource {resource.kind} with index {index}")
|
|
659
718
|
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
if ocean.metrics.sync_state != SyncState.FAILED:
|
|
663
|
-
ocean.metrics.sync_state = SyncState.COMPLETED
|
|
664
|
-
|
|
665
|
-
await ocean.metrics.send_metrics_to_webhook(
|
|
666
|
-
kind=resource_kind_id
|
|
667
|
-
)
|
|
668
|
-
# await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
|
|
719
|
+
creation_results.append(await self.process_resource(resource,index,user_agent_type))
|
|
669
720
|
|
|
670
721
|
await self.sort_and_upsert_failed_entities(user_agent_type)
|
|
671
722
|
|
|
@@ -721,3 +772,5 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
721
772
|
logger.info("Finished executing resync_complete hooks")
|
|
722
773
|
|
|
723
774
|
return True
|
|
775
|
+
finally:
|
|
776
|
+
await ocean.app.cache_provider.clear()
|
|
@@ -11,6 +11,16 @@ class CreatePortResourcesOrigin(StrEnum):
|
|
|
11
11
|
Port = "Port"
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
class ProcessExecutionMode(StrEnum):
|
|
15
|
+
multi_process = "multi_process"
|
|
16
|
+
single_process = "single_process"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CachingStorageMode(StrEnum):
|
|
20
|
+
disk = "disk"
|
|
21
|
+
memory = "memory"
|
|
22
|
+
|
|
23
|
+
|
|
14
24
|
class Runtime(Enum):
|
|
15
25
|
Saas = "Saas"
|
|
16
26
|
OnPrem = "OnPrem"
|
|
@@ -10,6 +10,7 @@ from prometheus_client import Gauge
|
|
|
10
10
|
import prometheus_client.openmetrics
|
|
11
11
|
import prometheus_client.openmetrics.exposition
|
|
12
12
|
import prometheus_client.parser
|
|
13
|
+
from prometheus_client import multiprocess
|
|
13
14
|
|
|
14
15
|
if TYPE_CHECKING:
|
|
15
16
|
from port_ocean.config.settings import MetricsSettings, IntegrationSettings
|
|
@@ -108,11 +109,14 @@ class Metrics:
|
|
|
108
109
|
metrics_settings: "MetricsSettings",
|
|
109
110
|
integration_configuration: "IntegrationSettings",
|
|
110
111
|
port_client: "PortClient",
|
|
112
|
+
multiprocessing_enabled: bool = False,
|
|
111
113
|
) -> None:
|
|
112
114
|
self.metrics_settings = metrics_settings
|
|
113
115
|
self.integration_configuration = integration_configuration
|
|
114
116
|
self.port_client = port_client
|
|
115
117
|
self.registry = prometheus_client.CollectorRegistry()
|
|
118
|
+
if multiprocessing_enabled:
|
|
119
|
+
multiprocess.MultiProcessCollector(self.registry)
|
|
116
120
|
self.metrics: dict[str, Gauge] = {}
|
|
117
121
|
self.load_metrics()
|
|
118
122
|
self._integration_version: Optional[str] = None
|
|
@@ -4,6 +4,10 @@ from contextlib import asynccontextmanager
|
|
|
4
4
|
import threading
|
|
5
5
|
from typing import Any, AsyncIterator, Callable, Dict, Type
|
|
6
6
|
|
|
7
|
+
from port_ocean.cache.base import CacheProvider
|
|
8
|
+
from port_ocean.cache.disk import DiskCacheProvider
|
|
9
|
+
from port_ocean.cache.memory import InMemoryCacheProvider
|
|
10
|
+
from port_ocean.core.models import ProcessExecutionMode
|
|
7
11
|
import port_ocean.helpers.metric.metric
|
|
8
12
|
|
|
9
13
|
from fastapi import FastAPI, APIRouter
|
|
@@ -66,11 +70,16 @@ class Ocean:
|
|
|
66
70
|
integration_type=self.config.integration.type,
|
|
67
71
|
integration_version=__integration_version__,
|
|
68
72
|
)
|
|
69
|
-
|
|
73
|
+
self.cache_provider: CacheProvider = self._get_caching_provider()
|
|
74
|
+
self.process_execution_mode: ProcessExecutionMode = (
|
|
75
|
+
self._get_process_execution_mode()
|
|
76
|
+
)
|
|
70
77
|
self.metrics = port_ocean.helpers.metric.metric.Metrics(
|
|
71
78
|
metrics_settings=self.config.metrics,
|
|
72
79
|
integration_configuration=self.config.integration,
|
|
73
80
|
port_client=self.port_client,
|
|
81
|
+
multiprocessing_enabled=self.process_execution_mode
|
|
82
|
+
== ProcessExecutionMode.multi_process,
|
|
74
83
|
)
|
|
75
84
|
|
|
76
85
|
self.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -90,6 +99,24 @@ class Ocean:
|
|
|
90
99
|
|
|
91
100
|
self.app_initialized = False
|
|
92
101
|
|
|
102
|
+
def _get_process_execution_mode(self) -> ProcessExecutionMode:
|
|
103
|
+
if self.config.process_execution_mode:
|
|
104
|
+
return self.config.process_execution_mode
|
|
105
|
+
return ProcessExecutionMode.single_process
|
|
106
|
+
|
|
107
|
+
def _get_caching_provider(self) -> CacheProvider:
|
|
108
|
+
if self.config.caching_storage_mode:
|
|
109
|
+
caching_type_to_provider = {
|
|
110
|
+
DiskCacheProvider.STORAGE_TYPE: DiskCacheProvider,
|
|
111
|
+
InMemoryCacheProvider.STORAGE_TYPE: InMemoryCacheProvider,
|
|
112
|
+
}
|
|
113
|
+
if self.config.caching_storage_mode in caching_type_to_provider:
|
|
114
|
+
return caching_type_to_provider[self.config.caching_storage_mode]()
|
|
115
|
+
|
|
116
|
+
if self.config.process_execution_mode == ProcessExecutionMode.multi_process:
|
|
117
|
+
return DiskCacheProvider()
|
|
118
|
+
return InMemoryCacheProvider()
|
|
119
|
+
|
|
93
120
|
def is_saas(self) -> bool:
|
|
94
121
|
return self.config.runtime.is_saas_runtime
|
|
95
122
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Tests for cache providers."""
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pytest
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from port_ocean.cache.disk import (
|
|
6
|
+
DiskCacheProvider,
|
|
7
|
+
FailedToReadCacheFileError,
|
|
8
|
+
FailedToWriteCacheFileError,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.fixture
|
|
13
|
+
def disk_cache(tmp_path: Path) -> DiskCacheProvider:
|
|
14
|
+
"""Fixture that provides a DiskCacheProvider with a temporary directory."""
|
|
15
|
+
return DiskCacheProvider(cache_dir=str(tmp_path))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.mark.asyncio
|
|
19
|
+
async def test_disk_cache_set_get(disk_cache: DiskCacheProvider) -> None:
|
|
20
|
+
"""Test setting and getting values from disk cache."""
|
|
21
|
+
# Test basic set/get
|
|
22
|
+
await disk_cache.set("test_key", "test_value")
|
|
23
|
+
assert await disk_cache.get("test_key") == "test_value"
|
|
24
|
+
|
|
25
|
+
# Test with different types
|
|
26
|
+
test_data = {
|
|
27
|
+
"string": "hello",
|
|
28
|
+
"int": 42,
|
|
29
|
+
"float": 3.14,
|
|
30
|
+
"list": [1, 2, 3],
|
|
31
|
+
"dict": {"a": 1, "b": 2},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
for key, value in test_data.items():
|
|
35
|
+
await disk_cache.set(key, value)
|
|
36
|
+
assert await disk_cache.get(key) == value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.asyncio
|
|
40
|
+
async def test_disk_cache_clear(disk_cache: DiskCacheProvider) -> None:
|
|
41
|
+
"""Test clearing all values from disk cache."""
|
|
42
|
+
# Add multiple values
|
|
43
|
+
for i in range(5):
|
|
44
|
+
await disk_cache.set(f"key_{i}", f"value_{i}")
|
|
45
|
+
|
|
46
|
+
# Verify values exist
|
|
47
|
+
for i in range(5):
|
|
48
|
+
assert await disk_cache.get(f"key_{i}") == f"value_{i}"
|
|
49
|
+
|
|
50
|
+
# Clear cache
|
|
51
|
+
await disk_cache.clear()
|
|
52
|
+
|
|
53
|
+
# Verify all values are gone
|
|
54
|
+
for i in range(5):
|
|
55
|
+
assert await disk_cache.get(f"key_{i}") is None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.mark.asyncio
|
|
59
|
+
async def test_disk_cache_nonexistent_key(disk_cache: DiskCacheProvider) -> None:
|
|
60
|
+
"""Test getting a nonexistent key from disk cache."""
|
|
61
|
+
assert await disk_cache.get("nonexistent_key") is None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.mark.asyncio
|
|
65
|
+
async def test_disk_cache_corrupted_file(
|
|
66
|
+
disk_cache: DiskCacheProvider, tmp_path: Path
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Test handling of corrupted cache files."""
|
|
69
|
+
# Create a corrupted pickle file
|
|
70
|
+
cache_path = tmp_path / "test_key.pkl"
|
|
71
|
+
with open(cache_path, "wb") as f:
|
|
72
|
+
f.write(b"invalid pickle data")
|
|
73
|
+
|
|
74
|
+
# Attempting to read should raise FailedToReadCacheFileError
|
|
75
|
+
with pytest.raises(FailedToReadCacheFileError):
|
|
76
|
+
await disk_cache.get("test_key")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@pytest.mark.asyncio
|
|
80
|
+
async def test_disk_cache_write_error(
|
|
81
|
+
disk_cache: DiskCacheProvider, tmp_path: Path
|
|
82
|
+
) -> None:
|
|
83
|
+
"""Test handling of write errors."""
|
|
84
|
+
# Make the cache directory read-only
|
|
85
|
+
os.chmod(tmp_path, 0o444)
|
|
86
|
+
|
|
87
|
+
# Attempting to write should raise FailedToWriteCacheFileError
|
|
88
|
+
with pytest.raises(FailedToWriteCacheFileError):
|
|
89
|
+
await disk_cache.set("test_key", "test_value")
|
|
90
|
+
|
|
91
|
+
# Restore permissions
|
|
92
|
+
os.chmod(tmp_path, 0o755)
|