port-ocean 0.24.22__tar.gz → 0.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {port_ocean-0.24.22 → port_ocean-0.25.0}/PKG-INFO +1 -1
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/sync_raw.py +307 -117
- {port_ocean-0.24.22 → port_ocean-0.25.0}/pyproject.toml +1 -1
- {port_ocean-0.24.22 → port_ocean-0.25.0}/LICENSE.md +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/README.md +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.Deb +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.alpine +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.base.builder +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.base.runner +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.dockerignore +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Dockerfile.local +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/Makefile +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/entry_local.sh +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/grpcio.sh +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/integrations/_infra/init.sh +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/bootstrap.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/disk.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/errors.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cache/memory.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cli.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/__init___.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/clean.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/dock.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/defaults/group.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/list_integrations.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/main.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/new.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/pull.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/sail.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/commands/version.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/cookiecutter.json +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/extensions.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/hooks/post_gen_project.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/debug.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/poetry.toml +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/sonar-project.properties +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/cli/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/auth_client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/auth/oauth_client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/authentication.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/blueprints.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/entities.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/integrations.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/migrations.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/mixins/organization.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/retry_transport.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/types.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/clients/port/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/dynamic.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/config/settings.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/consumers/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/consumers/kafka_consumer.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/event.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/metric_resource.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/ocean.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/context/resource.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/clean.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/common.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/defaults/initialize.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/factory.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/http.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/kafka.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/once.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/polling.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/event_listener/webhooks_only.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/applier.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/api.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/port_app_config/models.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/abstract_queue.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/queue/local_queue.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/resync_state_updater/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/resync_state_updater/updater.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/abstract_webhook_processor.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/processor_manager.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/handlers/webhook/webhook_event.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/events.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/handler.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/live_events.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/sync.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/integrations/mixins/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/models.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/ocean_types.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/utils/entity_topological_sorter.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/core/utils/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/debug_cli.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/api.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/clients.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/context.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/core.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/port_defaults.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/exceptions/webhook_processor.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/async_client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/metric/metric.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/metric/utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/helpers/retry.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/handlers.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/logger_setup.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/log/sensetive.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/middlewares.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/ocean.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/py.typed +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/run.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/sonar-project.properties +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/test_disk_cache.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/cache/test_memory_cache.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/oauth/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/oauth/test_oauth_client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_entities.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_integrations.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/clients/port/mixins/test_organization_mixin.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/config/test_config.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/conftest.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/conftest.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/defaults/test_common.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/mixins/test_live_events.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/port_app_config/test_api.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/port_app_config/test_base.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/queue/test_local_queue.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_processor_manager.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/handlers/webhook/test_webhook_event.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/test_utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/utils/test_entity_topological_sorter.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/core/utils/test_resolve_entities_diff.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/fake_port_api.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/fixtures.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/integration.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/ocean_app.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/port_client.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/helpers/smoke_test.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/log/test_handlers.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_metric.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_ocean.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/test_smoke.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/utils/test_async_iterators.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/tests/utils/test_cache.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/__init__.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/async_http.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/async_iterators.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/cache.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/ipc.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/misc.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/queue_utils.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/repeat.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/signal.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/utils/time.py +0 -0
- {port_ocean-0.24.22 → port_ocean-0.25.0}/port_ocean/version.py +0 -0
@@ -32,9 +32,21 @@ from port_ocean.core.ocean_types import (
|
|
32
32
|
RAW_ITEM,
|
33
33
|
CalculationResult,
|
34
34
|
)
|
35
|
-
from port_ocean.core.utils.utils import
|
36
|
-
|
37
|
-
|
35
|
+
from port_ocean.core.utils.utils import (
|
36
|
+
resolve_entities_diff,
|
37
|
+
zip_and_sum,
|
38
|
+
gather_and_split_errors_from_results,
|
39
|
+
)
|
40
|
+
from port_ocean.exceptions.core import (
|
41
|
+
IntegrationSubProcessFailedException,
|
42
|
+
OceanAbortException,
|
43
|
+
)
|
44
|
+
from port_ocean.helpers.metric.metric import (
|
45
|
+
MetricResourceKind,
|
46
|
+
SyncState,
|
47
|
+
MetricType,
|
48
|
+
MetricPhase,
|
49
|
+
)
|
38
50
|
from port_ocean.helpers.metric.utils import TimeMetric, TimeMetricWithResourceKind
|
39
51
|
from port_ocean.utils.ipc import FileIPC
|
40
52
|
|
@@ -101,10 +113,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
101
113
|
results = []
|
102
114
|
for task in fns:
|
103
115
|
if inspect.isasyncgenfunction(task):
|
104
|
-
logger.info(
|
116
|
+
logger.info(
|
117
|
+
f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
|
118
|
+
)
|
105
119
|
results.append(resync_generator_wrapper(task, resource_config.kind))
|
106
120
|
else:
|
107
|
-
logger.info(
|
121
|
+
logger.info(
|
122
|
+
f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
|
123
|
+
)
|
108
124
|
task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
|
109
125
|
tasks.append(resync_function_wrapper(task, resource_config.kind))
|
110
126
|
|
@@ -155,14 +171,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
155
171
|
{
|
156
172
|
"property": "$identifier",
|
157
173
|
"operator": "in",
|
158
|
-
"value": [entity.identifier for entity in entities]
|
174
|
+
"value": [entity.identifier for entity in entities],
|
159
175
|
},
|
160
176
|
{
|
161
177
|
"property": "$blueprint",
|
162
178
|
"operator": "=",
|
163
|
-
"value": entities[0].blueprint
|
164
|
-
}
|
165
|
-
]
|
179
|
+
"value": entities[0].blueprint,
|
180
|
+
},
|
181
|
+
],
|
166
182
|
}
|
167
183
|
|
168
184
|
async def _map_entities_compared_with_port(
|
@@ -174,7 +190,10 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
174
190
|
if not entities:
|
175
191
|
return []
|
176
192
|
|
177
|
-
if
|
193
|
+
if (
|
194
|
+
entities[0].is_using_search_identifier
|
195
|
+
or entities[0].is_using_search_relation
|
196
|
+
):
|
178
197
|
return entities
|
179
198
|
|
180
199
|
MIN_ENTITIES_TO_MAP = 10
|
@@ -186,21 +205,21 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
186
205
|
|
187
206
|
# Process entities in batches
|
188
207
|
for start_index in range(0, len(entities), BATCH_SIZE):
|
189
|
-
entities_batch = entities[start_index:start_index + BATCH_SIZE]
|
208
|
+
entities_batch = entities[start_index : start_index + BATCH_SIZE]
|
190
209
|
batch_results = await self._fetch_entities_batch_from_port(
|
191
|
-
entities_batch,
|
192
|
-
resource,
|
193
|
-
user_agent_type
|
210
|
+
entities_batch, resource, user_agent_type
|
194
211
|
)
|
195
212
|
entities_at_port_with_properties.extend(batch_results)
|
196
213
|
|
197
|
-
logger.info(
|
214
|
+
logger.info(
|
215
|
+
"Got entities from port with properties and relations",
|
216
|
+
port_entities=len(entities_at_port_with_properties),
|
217
|
+
)
|
198
218
|
|
199
219
|
if len(entities_at_port_with_properties) > 0:
|
200
220
|
return resolve_entities_diff(entities, entities_at_port_with_properties)
|
201
221
|
return entities
|
202
222
|
|
203
|
-
|
204
223
|
async def _fetch_entities_batch_from_port(
|
205
224
|
self,
|
206
225
|
entities_batch: list[Entity],
|
@@ -210,16 +229,18 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
210
229
|
query = self._construct_search_query_for_entities(entities_batch)
|
211
230
|
return await ocean.port_client.search_entities(
|
212
231
|
user_agent_type,
|
213
|
-
parameters_to_include=["blueprint", "identifier"]
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
]
|
220
|
-
|
232
|
+
parameters_to_include=["blueprint", "identifier"]
|
233
|
+
+ (["title"] if resource.port.entity.mappings.title != None else [])
|
234
|
+
+ (["team"] if resource.port.entity.mappings.team != None else [])
|
235
|
+
+ [
|
236
|
+
f"properties.{prop}"
|
237
|
+
for prop in resource.port.entity.mappings.properties
|
238
|
+
]
|
239
|
+
+ [
|
240
|
+
f"relations.{relation}"
|
241
|
+
for relation in resource.port.entity.mappings.relations
|
221
242
|
],
|
222
|
-
query=query
|
243
|
+
query=query,
|
223
244
|
)
|
224
245
|
|
225
246
|
async def _register_resource_raw(
|
@@ -228,7 +249,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
228
249
|
results: list[dict[Any, Any]],
|
229
250
|
user_agent_type: UserAgentType,
|
230
251
|
parse_all: bool = False,
|
231
|
-
send_raw_data_examples_amount: int = 0
|
252
|
+
send_raw_data_examples_amount: int = 0,
|
232
253
|
) -> CalculationResult:
|
233
254
|
objects_diff = await self._calculate_raw(
|
234
255
|
[(resource, results)], parse_all, send_raw_data_examples_amount
|
@@ -236,8 +257,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
236
257
|
|
237
258
|
ocean.metrics.inc_metric(
|
238
259
|
name=MetricType.OBJECT_COUNT_NAME,
|
239
|
-
labels=[
|
240
|
-
|
260
|
+
labels=[
|
261
|
+
ocean.metrics.current_resource_kind(),
|
262
|
+
MetricPhase.TRANSFORM,
|
263
|
+
MetricPhase.TransformResult.FAILED,
|
264
|
+
],
|
265
|
+
value=len(objects_diff[0].entity_selector_diff.failed),
|
241
266
|
)
|
242
267
|
|
243
268
|
modified_objects = []
|
@@ -247,43 +272,67 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
247
272
|
changed_entities = await self._map_entities_compared_with_port(
|
248
273
|
objects_diff[0].entity_selector_diff.passed,
|
249
274
|
resource,
|
250
|
-
user_agent_type
|
275
|
+
user_agent_type,
|
251
276
|
)
|
252
277
|
if changed_entities:
|
253
|
-
logger.info(
|
254
|
-
|
278
|
+
logger.info(
|
279
|
+
"Upserting changed entities",
|
280
|
+
changed_entities=len(changed_entities),
|
281
|
+
total_entities=len(objects_diff[0].entity_selector_diff.passed),
|
282
|
+
)
|
255
283
|
ocean.metrics.inc_metric(
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
284
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
285
|
+
labels=[
|
286
|
+
ocean.metrics.current_resource_kind(),
|
287
|
+
MetricPhase.LOAD,
|
288
|
+
MetricPhase.LoadResult.SKIPPED,
|
289
|
+
],
|
290
|
+
value=len(objects_diff[0].entity_selector_diff.passed)
|
291
|
+
- len(changed_entities),
|
292
|
+
)
|
260
293
|
await self.entities_state_applier.upsert(
|
261
294
|
changed_entities, user_agent_type
|
262
295
|
)
|
263
296
|
|
264
297
|
else:
|
265
|
-
logger.info(
|
298
|
+
logger.info(
|
299
|
+
"Entities in batch didn't changed since last sync, skipping",
|
300
|
+
total_entities=len(objects_diff[0].entity_selector_diff.passed),
|
301
|
+
)
|
266
302
|
ocean.metrics.inc_metric(
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
303
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
304
|
+
labels=[
|
305
|
+
ocean.metrics.current_resource_kind(),
|
306
|
+
MetricPhase.LOAD,
|
307
|
+
MetricPhase.LoadResult.SKIPPED,
|
308
|
+
],
|
309
|
+
value=len(objects_diff[0].entity_selector_diff.passed),
|
310
|
+
)
|
311
|
+
modified_objects = [
|
312
|
+
ocean.port_client._reduce_entity(entity)
|
313
|
+
for entity in objects_diff[0].entity_selector_diff.passed
|
314
|
+
]
|
272
315
|
except Exception as e:
|
273
|
-
logger.warning(
|
316
|
+
logger.warning(
|
317
|
+
f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}"
|
318
|
+
)
|
274
319
|
modified_objects = await self.entities_state_applier.upsert(
|
275
320
|
objects_diff[0].entity_selector_diff.passed, user_agent_type
|
276
|
-
|
321
|
+
)
|
277
322
|
else:
|
278
|
-
|
279
|
-
|
280
|
-
|
323
|
+
modified_objects = await self.entities_state_applier.upsert(
|
324
|
+
objects_diff[0].entity_selector_diff.passed, user_agent_type
|
325
|
+
)
|
281
326
|
|
282
327
|
return CalculationResult(
|
283
|
-
number_of_transformed_entities=len(
|
284
|
-
|
328
|
+
number_of_transformed_entities=len(
|
329
|
+
objects_diff[0].entity_selector_diff.passed
|
330
|
+
),
|
331
|
+
entity_selector_diff=objects_diff[0].entity_selector_diff._replace(
|
332
|
+
passed=modified_objects
|
333
|
+
),
|
285
334
|
errors=objects_diff[0].errors,
|
286
|
-
misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys
|
335
|
+
misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys,
|
287
336
|
)
|
288
337
|
|
289
338
|
async def _unregister_resource_raw(
|
@@ -334,17 +383,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
334
383
|
resource_config,
|
335
384
|
raw_results,
|
336
385
|
user_agent_type,
|
337
|
-
send_raw_data_examples_amount=send_raw_data_examples_amount
|
386
|
+
send_raw_data_examples_amount=send_raw_data_examples_amount,
|
338
387
|
)
|
339
388
|
errors.extend(calculation_result.errors)
|
340
389
|
passed_entities = list(calculation_result.entity_selector_diff.passed)
|
341
|
-
number_of_transformed_entities +=
|
390
|
+
number_of_transformed_entities += (
|
391
|
+
calculation_result.number_of_transformed_entities
|
392
|
+
)
|
342
393
|
logger.info(
|
343
394
|
f"Finished registering change for {len(raw_results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
|
344
395
|
)
|
345
396
|
|
346
|
-
|
347
|
-
|
348
397
|
for generator in async_generators:
|
349
398
|
try:
|
350
399
|
async for items in generator:
|
@@ -358,11 +407,15 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
358
407
|
resource_config,
|
359
408
|
items,
|
360
409
|
user_agent_type,
|
361
|
-
send_raw_data_examples_amount=send_raw_data_examples_amount
|
410
|
+
send_raw_data_examples_amount=send_raw_data_examples_amount,
|
411
|
+
)
|
412
|
+
passed_entities.extend(
|
413
|
+
calculation_result.entity_selector_diff.passed
|
362
414
|
)
|
363
|
-
passed_entities.extend(calculation_result.entity_selector_diff.passed)
|
364
415
|
errors.extend(calculation_result.errors)
|
365
|
-
number_of_transformed_entities +=
|
416
|
+
number_of_transformed_entities += (
|
417
|
+
calculation_result.number_of_transformed_entities
|
418
|
+
)
|
366
419
|
except* OceanAbortException as error:
|
367
420
|
ocean.metrics.sync_state = SyncState.FAILED
|
368
421
|
errors.append(error)
|
@@ -371,29 +424,40 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
371
424
|
f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
|
372
425
|
)
|
373
426
|
|
374
|
-
|
375
427
|
ocean.metrics.set_metric(
|
376
428
|
name=MetricType.SUCCESS_NAME,
|
377
429
|
labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
|
378
|
-
value=int(not errors)
|
430
|
+
value=int(not errors),
|
379
431
|
)
|
380
432
|
|
381
433
|
ocean.metrics.inc_metric(
|
382
434
|
name=MetricType.OBJECT_COUNT_NAME,
|
383
|
-
labels=[
|
384
|
-
|
435
|
+
labels=[
|
436
|
+
ocean.metrics.current_resource_kind(),
|
437
|
+
MetricPhase.EXTRACT,
|
438
|
+
MetricPhase.ExtractResult.EXTRACTED,
|
439
|
+
],
|
440
|
+
value=number_of_raw_results,
|
385
441
|
)
|
386
442
|
|
387
443
|
ocean.metrics.inc_metric(
|
388
444
|
name=MetricType.OBJECT_COUNT_NAME,
|
389
|
-
labels=[
|
390
|
-
|
445
|
+
labels=[
|
446
|
+
ocean.metrics.current_resource_kind(),
|
447
|
+
MetricPhase.TRANSFORM,
|
448
|
+
MetricPhase.TransformResult.TRANSFORMED,
|
449
|
+
],
|
450
|
+
value=number_of_transformed_entities,
|
391
451
|
)
|
392
452
|
|
393
453
|
ocean.metrics.inc_metric(
|
394
454
|
name=MetricType.OBJECT_COUNT_NAME,
|
395
|
-
labels=[
|
396
|
-
|
455
|
+
labels=[
|
456
|
+
ocean.metrics.current_resource_kind(),
|
457
|
+
MetricPhase.TRANSFORM,
|
458
|
+
MetricPhase.TransformResult.FILTERED_OUT,
|
459
|
+
],
|
460
|
+
value=number_of_raw_results - number_of_transformed_entities,
|
397
461
|
)
|
398
462
|
|
399
463
|
return passed_entities, errors
|
@@ -440,7 +504,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
440
504
|
errors = sum(errors, [])
|
441
505
|
misconfigured_entity_keys = list(misconfigured_entity_keys)
|
442
506
|
|
443
|
-
|
444
507
|
if errors:
|
445
508
|
message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
|
446
509
|
logger.error(message, exc_info=errors)
|
@@ -580,34 +643,54 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
580
643
|
user_agent_type,
|
581
644
|
)
|
582
645
|
|
583
|
-
async def sort_and_upsert_failed_entities(
|
646
|
+
async def sort_and_upsert_failed_entities(
|
647
|
+
self, user_agent_type: UserAgentType
|
648
|
+
) -> None:
|
584
649
|
try:
|
585
650
|
if not event.entity_topological_sorter.should_execute():
|
586
651
|
return None
|
587
|
-
logger.info(
|
652
|
+
logger.info(
|
653
|
+
f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",
|
654
|
+
failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count(),
|
655
|
+
)
|
588
656
|
|
589
657
|
for entity in event.entity_topological_sorter.get_entities():
|
590
|
-
await self.entities_state_applier.context.port_client.upsert_entity(
|
658
|
+
await self.entities_state_applier.context.port_client.upsert_entity(
|
659
|
+
entity,
|
660
|
+
event.port_app_config.get_port_request_options(),
|
661
|
+
user_agent_type,
|
662
|
+
should_raise=False,
|
663
|
+
)
|
591
664
|
|
592
665
|
except OceanAbortException as ocean_abort:
|
593
|
-
logger.info(
|
594
|
-
|
666
|
+
logger.info(
|
667
|
+
f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",
|
668
|
+
failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count(),
|
669
|
+
)
|
670
|
+
if isinstance(ocean_abort.__cause__, CycleError):
|
595
671
|
for entity in event.entity_topological_sorter.get_entities(False):
|
596
|
-
await self.entities_state_applier.context.port_client.upsert_entity(
|
672
|
+
await self.entities_state_applier.context.port_client.upsert_entity(
|
673
|
+
entity,
|
674
|
+
event.port_app_config.get_port_request_options(),
|
675
|
+
user_agent_type,
|
676
|
+
should_raise=False,
|
677
|
+
)
|
597
678
|
|
598
|
-
def process_resource_in_subprocess(
|
679
|
+
def process_resource_in_subprocess(
|
680
|
+
self,
|
599
681
|
file_ipc_map: dict[str, FileIPC],
|
600
682
|
resource: ResourceConfig,
|
601
683
|
index: int,
|
602
684
|
user_agent_type: UserAgentType,
|
603
685
|
) -> None:
|
604
|
-
logger.info(
|
686
|
+
logger.info(
|
687
|
+
f"process started successfully for {resource.kind} with index {index}"
|
688
|
+
)
|
605
689
|
|
606
690
|
clear_http_client_context()
|
691
|
+
|
607
692
|
async def process_resource_task() -> None:
|
608
|
-
result = await self._process_resource(
|
609
|
-
resource, index, user_agent_type
|
610
|
-
)
|
693
|
+
result = await self._process_resource(resource, index, user_agent_type)
|
611
694
|
file_ipc_map["process_resource"].save(result)
|
612
695
|
file_ipc_map["topological_entities"].save(
|
613
696
|
event.entity_topological_sorter.entities
|
@@ -616,32 +699,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
616
699
|
asyncio.run(process_resource_task())
|
617
700
|
logger.info(f"Process finished for {resource.kind} with index {index}")
|
618
701
|
|
619
|
-
async def
|
620
|
-
|
621
|
-
|
622
|
-
id = uuid.uuid4()
|
623
|
-
logger.info(f"Starting subprocess with id {id}")
|
624
|
-
file_ipc_map = {
|
625
|
-
"process_resource": FileIPC(id, "process_resource",([],[IntegrationSubProcessFailedException(f"Subprocess failed for {resource.kind} with index {index}")])),
|
626
|
-
"topological_entities": FileIPC(id, "topological_entities",[]),
|
627
|
-
}
|
628
|
-
process = ProcessWrapper(target=self.process_resource_in_subprocess, args=(file_ipc_map,resource,index,user_agent_type))
|
629
|
-
process.start()
|
630
|
-
await process.join_async()
|
631
|
-
|
632
|
-
event.entity_topological_sorter.entities.extend(file_ipc_map["topological_entities"].load())
|
633
|
-
return file_ipc_map["process_resource"].load()
|
634
|
-
|
635
|
-
else:
|
636
|
-
return await self._process_resource(resource,index,user_agent_type)
|
637
|
-
|
638
|
-
async def _process_resource(self,resource: ResourceConfig, index: int, user_agent_type: UserAgentType)-> tuple[list[Entity], list[Exception]]:
|
702
|
+
async def _process_resource(
|
703
|
+
self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
|
704
|
+
) -> tuple[list[Entity], list[Exception]]:
|
639
705
|
# create resource context per resource kind, so resync method could have access to the resource
|
640
706
|
# config as we might have multiple resources in the same event
|
641
|
-
async with resource_context(resource,index):
|
707
|
+
async with resource_context(resource, index):
|
642
708
|
resource_kind_id = f"{resource.kind}-{index}"
|
643
709
|
ocean.metrics.sync_state = SyncState.SYNCING
|
644
|
-
await ocean.metrics.report_kind_sync_metrics(
|
710
|
+
await ocean.metrics.report_kind_sync_metrics(
|
711
|
+
kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
|
712
|
+
)
|
645
713
|
|
646
714
|
task = asyncio.create_task(
|
647
715
|
self._register_in_batches(resource, user_agent_type)
|
@@ -652,22 +720,87 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
652
720
|
if ocean.metrics.sync_state != SyncState.FAILED:
|
653
721
|
ocean.metrics.sync_state = SyncState.COMPLETED
|
654
722
|
|
655
|
-
await ocean.metrics.send_metrics_to_webhook(
|
656
|
-
|
723
|
+
await ocean.metrics.send_metrics_to_webhook(kind=resource_kind_id)
|
724
|
+
await ocean.metrics.report_kind_sync_metrics(
|
725
|
+
kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
|
657
726
|
)
|
658
|
-
await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint)
|
659
727
|
|
660
728
|
return kind_results
|
661
729
|
|
662
|
-
|
663
|
-
async def resync_reconciliation(
|
730
|
+
def resync_reconciliation_in_subprocess(
|
664
731
|
self,
|
732
|
+
file_ipc_map: dict[str, FileIPC],
|
665
733
|
creation_results: list[tuple[list[Entity], list[Exception]]],
|
666
734
|
did_fetched_current_state: bool,
|
667
735
|
user_agent_type: UserAgentType,
|
668
736
|
app_config: Any,
|
669
737
|
silent: bool = True,
|
670
738
|
) -> None:
|
739
|
+
logger.info("Resync reconciliation subprocess started successfully")
|
740
|
+
|
741
|
+
clear_http_client_context()
|
742
|
+
|
743
|
+
async def resync_reconciliation_task() -> None:
|
744
|
+
result = await self._resync_reconciliation(
|
745
|
+
creation_results,
|
746
|
+
did_fetched_current_state,
|
747
|
+
user_agent_type,
|
748
|
+
app_config,
|
749
|
+
silent,
|
750
|
+
)
|
751
|
+
file_ipc_map["resync_reconciliation"].save(result)
|
752
|
+
|
753
|
+
asyncio.run(resync_reconciliation_task())
|
754
|
+
logger.info("Resync reconciliation subprocess finished")
|
755
|
+
|
756
|
+
async def process_resource(
|
757
|
+
self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
|
758
|
+
) -> tuple[list[Entity], list[Exception]]:
|
759
|
+
with logger.contextualize(resource_kind=resource.kind, index=index):
|
760
|
+
if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
|
761
|
+
id = uuid.uuid4()
|
762
|
+
logger.info(f"Starting subprocess with id {id}")
|
763
|
+
file_ipc_map = {
|
764
|
+
"process_resource": FileIPC(
|
765
|
+
str(id),
|
766
|
+
"process_resource",
|
767
|
+
(
|
768
|
+
[],
|
769
|
+
[
|
770
|
+
IntegrationSubProcessFailedException(
|
771
|
+
f"Subprocess failed for {resource.kind} with index {index}"
|
772
|
+
)
|
773
|
+
],
|
774
|
+
),
|
775
|
+
),
|
776
|
+
"topological_entities": FileIPC(
|
777
|
+
str(id), "topological_entities", []
|
778
|
+
),
|
779
|
+
}
|
780
|
+
process = ProcessWrapper(
|
781
|
+
target=self.process_resource_in_subprocess,
|
782
|
+
args=(file_ipc_map, resource, index, user_agent_type),
|
783
|
+
)
|
784
|
+
process.start()
|
785
|
+
await process.join_async()
|
786
|
+
|
787
|
+
event.entity_topological_sorter.entities.extend(
|
788
|
+
file_ipc_map["topological_entities"].load()
|
789
|
+
)
|
790
|
+
return file_ipc_map["process_resource"].load()
|
791
|
+
|
792
|
+
else:
|
793
|
+
return await self._process_resource(resource, index, user_agent_type)
|
794
|
+
|
795
|
+
@TimeMetricWithResourceKind(MetricPhase.RESYNC)
|
796
|
+
async def _resync_reconciliation(
|
797
|
+
self,
|
798
|
+
creation_results: list[tuple[list[Entity], list[Exception]]],
|
799
|
+
did_fetched_current_state: bool,
|
800
|
+
user_agent_type: UserAgentType,
|
801
|
+
app_config: Any,
|
802
|
+
silent: bool = True,
|
803
|
+
) -> bool:
|
671
804
|
"""Handle the reconciliation phase of the resync process.
|
672
805
|
|
673
806
|
This method handles:
|
@@ -716,13 +849,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
716
849
|
logger.info(
|
717
850
|
f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
|
718
851
|
)
|
719
|
-
entities_at_port = await ocean.port_client.search_entities(
|
720
|
-
user_agent_type
|
721
|
-
)
|
852
|
+
entities_at_port = await ocean.port_client.search_entities(user_agent_type)
|
722
853
|
|
723
854
|
await self.entities_state_applier.delete_diff(
|
724
855
|
{"before": entities_at_port, "after": generated_entities},
|
725
|
-
user_agent_type,
|
856
|
+
user_agent_type,
|
857
|
+
app_config.get_entity_deletion_threshold(),
|
726
858
|
)
|
727
859
|
|
728
860
|
logger.info("Resync finished successfully")
|
@@ -738,6 +870,47 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
738
870
|
|
739
871
|
return True
|
740
872
|
|
873
|
+
async def resync_reconciliation(
|
874
|
+
self,
|
875
|
+
creation_results: list[tuple[list[Entity], list[Exception]]],
|
876
|
+
did_fetched_current_state: bool,
|
877
|
+
user_agent_type: UserAgentType,
|
878
|
+
app_config: Any,
|
879
|
+
silent: bool = True,
|
880
|
+
) -> bool:
|
881
|
+
if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
|
882
|
+
id = uuid.uuid4()
|
883
|
+
logger.info(f"Starting resync reconciliation in subprocess with id {id}")
|
884
|
+
|
885
|
+
file_ipc_map = {
|
886
|
+
"resync_reconciliation": FileIPC(
|
887
|
+
str(id), "resync_reconciliation", False
|
888
|
+
),
|
889
|
+
}
|
890
|
+
|
891
|
+
process = ProcessWrapper(
|
892
|
+
target=self.resync_reconciliation_in_subprocess,
|
893
|
+
args=(
|
894
|
+
file_ipc_map,
|
895
|
+
creation_results,
|
896
|
+
did_fetched_current_state,
|
897
|
+
user_agent_type,
|
898
|
+
app_config,
|
899
|
+
silent,
|
900
|
+
),
|
901
|
+
)
|
902
|
+
process.start()
|
903
|
+
await process.join_async()
|
904
|
+
|
905
|
+
return file_ipc_map["resync_reconciliation"].load()
|
906
|
+
else:
|
907
|
+
return await self._resync_reconciliation(
|
908
|
+
creation_results,
|
909
|
+
did_fetched_current_state,
|
910
|
+
user_agent_type,
|
911
|
+
app_config,
|
912
|
+
silent,
|
913
|
+
)
|
741
914
|
|
742
915
|
@TimeMetric(MetricPhase.RESYNC)
|
743
916
|
async def sync_raw_all(
|
@@ -773,8 +946,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
773
946
|
)
|
774
947
|
logger.info(f"Resync will use the following mappings: {app_config.dict()}")
|
775
948
|
|
776
|
-
kinds = [
|
777
|
-
|
949
|
+
kinds = [
|
950
|
+
f"{resource.kind}-{index}"
|
951
|
+
for index, resource in enumerate(app_config.resources)
|
952
|
+
]
|
953
|
+
blueprints = [
|
954
|
+
resource.port.entity.mappings.blueprint
|
955
|
+
for resource in app_config.resources
|
956
|
+
]
|
778
957
|
ocean.metrics.initialize_metrics(kinds)
|
779
958
|
await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
|
780
959
|
|
@@ -799,13 +978,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
799
978
|
|
800
979
|
creation_results: list[tuple[list[Entity], list[Exception]]] = []
|
801
980
|
|
802
|
-
multiprocessing.set_start_method(
|
981
|
+
multiprocessing.set_start_method("fork", True)
|
803
982
|
try:
|
804
|
-
for index,resource in enumerate(app_config.resources):
|
805
|
-
logger.info(
|
806
|
-
|
983
|
+
for index, resource in enumerate(app_config.resources):
|
984
|
+
logger.info(
|
985
|
+
f"Starting processing resource {resource.kind} with index {index}"
|
986
|
+
)
|
987
|
+
creation_results.append(
|
988
|
+
await self.process_resource(resource, index, user_agent_type)
|
989
|
+
)
|
807
990
|
except asyncio.CancelledError as e:
|
808
|
-
logger.warning(
|
991
|
+
logger.warning(
|
992
|
+
"Resync aborted successfully, skipping delete phase. This leads to an incomplete state"
|
993
|
+
)
|
809
994
|
raise
|
810
995
|
else:
|
811
996
|
success = await self.resync_reconciliation(
|
@@ -813,11 +998,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
813
998
|
did_fetched_current_state,
|
814
999
|
user_agent_type,
|
815
1000
|
app_config,
|
816
|
-
silent
|
1001
|
+
silent,
|
1002
|
+
)
|
1003
|
+
await ocean.metrics.report_sync_metrics(
|
1004
|
+
kinds=[MetricResourceKind.RECONCILIATION]
|
817
1005
|
)
|
818
|
-
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
|
819
1006
|
return success
|
820
1007
|
finally:
|
821
1008
|
await ocean.app.cache_provider.clear()
|
822
|
-
if
|
1009
|
+
if (
|
1010
|
+
ocean.app.process_execution_mode
|
1011
|
+
== ProcessExecutionMode.multi_process
|
1012
|
+
):
|
823
1013
|
ocean.metrics.cleanup_prometheus_metrics()
|