howler-api 4.1.0.dev1389__tar.gz → 4.1.0.dev1418__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/PKG-INFO +1 -1
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/transition.py +32 -10
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/action.py +2 -2
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/hit.py +19 -14
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/ingest.py +2 -2
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/collection.py +87 -9
- howler_api-4.1.0.dev1418/howler/external/check_duplicates.py +152 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/mixins.py +10 -4
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/case_service.py +11 -10
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/dossier_service.py +2 -2
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/hit_service.py +31 -16
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/compat.py +3 -3
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/pyproject.toml +1 -1
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_label.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_to_bundle.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_to_case.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/change_field.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/demote.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/example_plugin.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/prioritization.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/promote.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/remove_from_bundle.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/remove_label.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/base.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/socket.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/analytic.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/auth.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/clue.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/configs.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/dossier.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/help.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/notebook.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/overview.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/search.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/template.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/tool.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/user.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/etag.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/params.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/view.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/case.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/fuzzy.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/search.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/app.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/classification.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/classification.yml +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/exceptions.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/loader.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/audit.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/format.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/net.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/net_static.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/random_user.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/swagger.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/config.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/action_queue_worker.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/correlation.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/retention.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/view_cleanup.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/bulk.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/constants.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/exceptions.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/howler_store.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/migrations/fix_process.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/operations.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/schemas.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/store.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/build.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/schemas.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/types.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/error.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_mitre.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_sigma_rules.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_tlds.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/reindex_data.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/wipe_databases.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/gunicorn_config.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/healthz.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/azure.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/discover.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/hit.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/oauth.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/search.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/workflow.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/ws.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/base.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/charter.txt +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/constants.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/helper.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/howler_enum.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/action.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/analytic.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/assemblyline.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/aws.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/azure.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/case.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/cbs.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/clue.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/config.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/dossier.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/agent.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/autonomous_system.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/client.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/cloud.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/code_signature.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/container.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/dns.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/egress.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/elf.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/email.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/error.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/event.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/faas.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/file.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/geo.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/group.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/hash.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/host.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/http.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/ingress.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/interface.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/network.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/observer.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/organization.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/os.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/pe.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/process.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/registry.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/related.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/rule.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/server.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/threat.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/tls.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/url.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/user.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/user_agent.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/vulnerability.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/event.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/gcp.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/hit.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/howler_data.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/lead.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/localized_label.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/overview.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/pivot.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/record.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/template.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/user.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/view.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/random_data.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/randomizer.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/patched.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/plugins/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/plugins/config.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/README.md +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/counters.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/events.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/hash.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/lock.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/comms.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/multi.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/named.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/priority.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/set.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/user_quota_tracker.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/socket.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/action_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/analytic_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/auth_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/bundle_compat_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/comms_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/config_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/correlation_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/docs_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/event_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/fuzzy_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/jwt_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/lucene_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/notebook_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/overview_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/search_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/template_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/user_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/viewer_service.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/telemetry.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/__init__.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/annotations.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/chunk.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/constants.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/dict_utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/indexes.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/isotime.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/list_utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/lucene.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/net_utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/path.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/socket_utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/str_utils.py +0 -0
- {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/uid.py +0 -0
|
@@ -5,6 +5,7 @@ from howler.actions import check_hit_limit
|
|
|
5
5
|
from howler.common.exceptions import InvalidDataException, NotFoundException
|
|
6
6
|
from howler.common.loader import datastore
|
|
7
7
|
from howler.common.logging import get_logger
|
|
8
|
+
from howler.datastore.exceptions import VersionConflictException
|
|
8
9
|
from howler.helper.workflow import Workflow, WorkflowException
|
|
9
10
|
from howler.odm.models.action import VALID_TRIGGERS
|
|
10
11
|
from howler.odm.models.howler_data import (
|
|
@@ -20,11 +21,39 @@ from howler.utils.list_utils import flatten_list
|
|
|
20
21
|
OPERATION_ID = "transition"
|
|
21
22
|
MAX_HITS_BASIC = 10
|
|
22
23
|
MAX_HITS_ADVANCED = 1000
|
|
24
|
+
MAX_VERSION_CONFLICT_ATTEMPTS = 3
|
|
23
25
|
SKIP_CENTRAL_LIMIT = True # This operation transforms the query, handles limit check locally
|
|
24
26
|
|
|
25
27
|
log = get_logger(__file__)
|
|
26
28
|
|
|
27
29
|
|
|
30
|
+
def _transition_failure_report(hit_id: str, error: Exception) -> dict[str, str]:
|
|
31
|
+
if isinstance(error, VersionConflictException):
|
|
32
|
+
return {
|
|
33
|
+
"query": f"howler.id:{hit_id}",
|
|
34
|
+
"outcome": "error",
|
|
35
|
+
"title": "Version Conflict",
|
|
36
|
+
"message": "The hit was modified while this transition was running.",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
"query": f"howler.id:{hit_id}",
|
|
41
|
+
"outcome": "error",
|
|
42
|
+
"title": "An error occurred while processing.",
|
|
43
|
+
"message": str(error),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _transition_hit_with_retry(hit_id: str, transition: HitStatusTransition, user: User, **kwargs) -> None:
|
|
48
|
+
for attempt in range(MAX_VERSION_CONFLICT_ATTEMPTS):
|
|
49
|
+
try:
|
|
50
|
+
hit_service.transition_hit(hit_id, transition, user, **kwargs)
|
|
51
|
+
return
|
|
52
|
+
except VersionConflictException:
|
|
53
|
+
if attempt == MAX_VERSION_CONFLICT_ATTEMPTS - 1:
|
|
54
|
+
raise
|
|
55
|
+
|
|
56
|
+
|
|
28
57
|
def __parse_workflow_actions(workflow: Workflow) -> dict[str, set[str]]:
|
|
29
58
|
"""Take in a workflow, and parse the steps and transitions of that workflow into a format understood by the UI"""
|
|
30
59
|
parsed_args: dict[str, set[str]] = {}
|
|
@@ -126,22 +155,15 @@ def execute(
|
|
|
126
155
|
total_processed = 0
|
|
127
156
|
for hit_id in ids:
|
|
128
157
|
try:
|
|
129
|
-
|
|
158
|
+
_transition_hit_with_retry(
|
|
130
159
|
hit_id,
|
|
131
160
|
cast(HitStatusTransition, HitStatusTransition[transition]),
|
|
132
161
|
user,
|
|
133
162
|
**kwargs,
|
|
134
163
|
)
|
|
135
164
|
success_ids.add(hit_id)
|
|
136
|
-
except (InvalidDataException, NotFoundException, WorkflowException) as e:
|
|
137
|
-
report.append(
|
|
138
|
-
{
|
|
139
|
-
"query": f"howler.id:{hit_id}",
|
|
140
|
-
"outcome": "error",
|
|
141
|
-
"title": "An error occurred while processing.",
|
|
142
|
-
"message": str(e),
|
|
143
|
-
}
|
|
144
|
-
)
|
|
165
|
+
except (InvalidDataException, NotFoundException, VersionConflictException, WorkflowException) as e:
|
|
166
|
+
report.append(_transition_failure_report(hit_id, e))
|
|
145
167
|
|
|
146
168
|
total_processed += 1
|
|
147
169
|
if total_processed % 10 == 0:
|
|
@@ -143,7 +143,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
|
|
|
143
143
|
|
|
144
144
|
ds = datastore()
|
|
145
145
|
|
|
146
|
-
existing_action = ds.action.get(id, as_obj=False)
|
|
146
|
+
existing_action, server_version = ds.action.get(id, as_obj=False, version=True)
|
|
147
147
|
|
|
148
148
|
if not existing_action:
|
|
149
149
|
return not_found(err="The specified automation does not exist")
|
|
@@ -166,7 +166,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
|
|
|
166
166
|
action_obj = Action(updated_action)
|
|
167
167
|
action_obj.action_id = id
|
|
168
168
|
|
|
169
|
-
ds.action.save(action_obj.action_id, action_obj, refresh=refresh)
|
|
169
|
+
ds.action.save(action_obj.action_id, action_obj, version=server_version, refresh=refresh)
|
|
170
170
|
except HowlerException as e:
|
|
171
171
|
return bad_request(err=str(e))
|
|
172
172
|
|
|
@@ -528,7 +528,7 @@ def get_assigned_hits(user, **kwargs):
|
|
|
528
528
|
@api_login(audit=False, required_priv=["W"])
|
|
529
529
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
530
530
|
@parse_parameters(refresh=parse_refresh)
|
|
531
|
-
def add_label(id, label_set, user, **kwargs):
|
|
531
|
+
def add_label(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
|
|
532
532
|
"""Add labels to a hit.
|
|
533
533
|
|
|
534
534
|
Variables:
|
|
@@ -575,7 +575,8 @@ def add_label(id, label_set, user, **kwargs):
|
|
|
575
575
|
hit_service.update_hit(
|
|
576
576
|
id,
|
|
577
577
|
[hit_helper.list_add(f"howler.labels.{label_set}", label) for label in labels],
|
|
578
|
-
user
|
|
578
|
+
user.uname,
|
|
579
|
+
version=server_version,
|
|
579
580
|
refresh="true" if DEBUG_FORCE_REFRESH else refresh,
|
|
580
581
|
)
|
|
581
582
|
|
|
@@ -591,7 +592,7 @@ def add_label(id, label_set, user, **kwargs):
|
|
|
591
592
|
@api_login(audit=False, required_priv=["W"])
|
|
592
593
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
593
594
|
@parse_parameters(refresh=parse_refresh)
|
|
594
|
-
def remove_labels(id, label_set, user, **kwargs):
|
|
595
|
+
def remove_labels(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
|
|
595
596
|
"""Remove labels from a hit.
|
|
596
597
|
|
|
597
598
|
Variables:
|
|
@@ -632,7 +633,8 @@ def remove_labels(id, label_set, user, **kwargs):
|
|
|
632
633
|
hit_service.update_hit(
|
|
633
634
|
id,
|
|
634
635
|
[hit_helper.list_remove(f"howler.labels.{label_set}", label) for label in labels],
|
|
635
|
-
user
|
|
636
|
+
user.uname,
|
|
637
|
+
version=server_version,
|
|
636
638
|
refresh="true" if DEBUG_FORCE_REFRESH else refresh,
|
|
637
639
|
)
|
|
638
640
|
|
|
@@ -648,7 +650,7 @@ def remove_labels(id, label_set, user, **kwargs):
|
|
|
648
650
|
@api_login(audit=False, required_priv=["W"])
|
|
649
651
|
@add_etag(getter=hit_service.get_hit, check_if_match=True)
|
|
650
652
|
@parse_parameters(refresh=parse_refresh)
|
|
651
|
-
def transition(id: str, user: User, **kwargs):
|
|
653
|
+
def transition(id: str, user: User, server_version: str | None = None, **kwargs):
|
|
652
654
|
"""Transition a hit
|
|
653
655
|
|
|
654
656
|
Variables:
|
|
@@ -679,11 +681,12 @@ def transition(id: str, user: User, **kwargs):
|
|
|
679
681
|
return bad_request(err="Invalid data format")
|
|
680
682
|
|
|
681
683
|
transition = transition_data["transition"]
|
|
684
|
+
version: str | None = None
|
|
682
685
|
if "If-Match" in request.headers:
|
|
683
686
|
version = request.headers["If-Match"]
|
|
684
687
|
else:
|
|
685
688
|
logger.warning("User is mising version - no If-Match header in request.")
|
|
686
|
-
version =
|
|
689
|
+
version = server_version
|
|
687
690
|
|
|
688
691
|
try:
|
|
689
692
|
if transition not in HitStatusTransition.list():
|
|
@@ -694,7 +697,7 @@ def transition(id: str, user: User, **kwargs):
|
|
|
694
697
|
)
|
|
695
698
|
)
|
|
696
699
|
|
|
697
|
-
hit_service.transition_hit(
|
|
700
|
+
hit, version = hit_service.transition_hit(
|
|
698
701
|
id, transition, user, version, refresh=refresh, **kwargs, **transition_data.get("data", {})
|
|
699
702
|
)
|
|
700
703
|
except (WorkflowException, DataStoreException, InvalidDataException) as e:
|
|
@@ -704,7 +707,6 @@ def transition(id: str, user: User, **kwargs):
|
|
|
704
707
|
except HowlerException as e:
|
|
705
708
|
return internal_error(err=str(e))
|
|
706
709
|
|
|
707
|
-
hit, version = hit_service.get_hit(id, as_odm=False, version=True)
|
|
708
710
|
return ok(hit), version
|
|
709
711
|
|
|
710
712
|
|
|
@@ -712,7 +714,7 @@ def transition(id: str, user: User, **kwargs):
|
|
|
712
714
|
@hit_api.route("/<id>/comments/<comment_id>", methods=["GET"])
|
|
713
715
|
@api_login(audit=False, required_priv=["R"])
|
|
714
716
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
715
|
-
def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwargs):
|
|
717
|
+
def get_comment(id: str, comment_id: str, user: User, server_version: str | None = None, **kwargs):
|
|
716
718
|
"""Get a comment associated with a particular hit
|
|
717
719
|
|
|
718
720
|
Variables:
|
|
@@ -741,7 +743,7 @@ def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwa
|
|
|
741
743
|
@hit_api.route("/<id>/comments", methods=["POST"])
|
|
742
744
|
@api_login(audit=False, required_priv=["W"])
|
|
743
745
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
744
|
-
def add_comment(id: str, user:
|
|
746
|
+
def add_comment(id: str, user: User, server_version: str | None = None, **kwargs):
|
|
745
747
|
"""Add a comment
|
|
746
748
|
|
|
747
749
|
Variables:
|
|
@@ -781,12 +783,13 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
|
|
|
781
783
|
[
|
|
782
784
|
hit_helper.list_add(
|
|
783
785
|
"howler.comment",
|
|
784
|
-
Comment({"user": user
|
|
786
|
+
Comment({"user": user.uname, "value": comment_value}),
|
|
785
787
|
explanation=f"Added a comment:\n\n{comment_value}",
|
|
786
788
|
if_missing=True,
|
|
787
789
|
),
|
|
788
790
|
],
|
|
789
|
-
user
|
|
791
|
+
user.uname,
|
|
792
|
+
version=server_version,
|
|
790
793
|
)
|
|
791
794
|
except DataStoreException as e:
|
|
792
795
|
return bad_request(err=str(e))
|
|
@@ -800,7 +803,7 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
|
|
|
800
803
|
@hit_api.route("/<id>/comments/<comment_id>", methods=["PUT"])
|
|
801
804
|
@api_login(audit=False, required_priv=["W"])
|
|
802
805
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
803
|
-
def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
|
|
806
|
+
def edit_comment(id: str, comment_id: str, user: dict[str, Any], server_version: str | None = None, **kwargs):
|
|
804
807
|
"""Edit a comment
|
|
805
808
|
|
|
806
809
|
Variables:
|
|
@@ -865,6 +868,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
|
|
|
865
868
|
),
|
|
866
869
|
],
|
|
867
870
|
user["uname"],
|
|
871
|
+
version=server_version,
|
|
868
872
|
)
|
|
869
873
|
|
|
870
874
|
return ok(new_hit), version
|
|
@@ -874,7 +878,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
|
|
|
874
878
|
@hit_api.route("/<id>/comments", methods=["DELETE"])
|
|
875
879
|
@api_login(audit=False, required_priv=["W"])
|
|
876
880
|
@add_etag(getter=hit_service.get_hit, check_if_match=False)
|
|
877
|
-
def delete_comments(id: str, user: User, **kwargs):
|
|
881
|
+
def delete_comments(id: str, user: User, server_version: str | None = None, **kwargs):
|
|
878
882
|
"""Delete a set of comments
|
|
879
883
|
|
|
880
884
|
Variables:
|
|
@@ -923,6 +927,7 @@ def delete_comments(id: str, user: User, **kwargs):
|
|
|
923
927
|
for comment in comments
|
|
924
928
|
],
|
|
925
929
|
user.uname,
|
|
930
|
+
version=server_version,
|
|
926
931
|
)
|
|
927
932
|
|
|
928
933
|
except DataStoreException as e:
|
|
@@ -265,7 +265,7 @@ def overwrite(index: str, id: str, **kwargs):
|
|
|
265
265
|
|
|
266
266
|
ds = datastore()
|
|
267
267
|
|
|
268
|
-
record = ds[index].get(id, as_obj=False, version=
|
|
268
|
+
record, server_version = ds[index].get(id, as_obj=False, version=True)
|
|
269
269
|
if not record:
|
|
270
270
|
return not_found(err="Record %s does not exist" % id)
|
|
271
271
|
|
|
@@ -292,7 +292,7 @@ def overwrite(index: str, id: str, **kwargs):
|
|
|
292
292
|
ds[index].save(
|
|
293
293
|
id,
|
|
294
294
|
odm(new_record) if odm else new_record,
|
|
295
|
-
version=
|
|
295
|
+
version=server_version,
|
|
296
296
|
refresh=refresh,
|
|
297
297
|
)
|
|
298
298
|
|
|
@@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, Generic, Literal, Optional, TypeVar, Uni
|
|
|
16
16
|
import elasticsearch
|
|
17
17
|
from datemath import dm
|
|
18
18
|
from datemath.helpers import DateMathException
|
|
19
|
+
from elastic_transport import ApiResponseMeta
|
|
19
20
|
from opentelemetry import trace
|
|
20
21
|
|
|
21
22
|
from howler import odm
|
|
@@ -1136,13 +1137,46 @@ class ESCollection(Generic[ModelType]):
|
|
|
1136
1137
|
|
|
1137
1138
|
return data
|
|
1138
1139
|
|
|
1140
|
+
def _search_exists(self, key) -> bool:
|
|
1141
|
+
"""Check document existence with an alias-safe search query."""
|
|
1142
|
+
result = self.with_retries(
|
|
1143
|
+
self.datastore.client.search,
|
|
1144
|
+
index=self.name,
|
|
1145
|
+
query={"ids": {"values": [key]}},
|
|
1146
|
+
size=0,
|
|
1147
|
+
track_total_hits=True,
|
|
1148
|
+
)
|
|
1149
|
+
total = result["hits"]["total"]
|
|
1150
|
+
return total["value"] > 0 if isinstance(total, dict) else total > 0
|
|
1151
|
+
|
|
1139
1152
|
def exists(self, key) -> bool:
|
|
1140
1153
|
"""Check if a document exists in the datastore.
|
|
1141
1154
|
|
|
1142
1155
|
:param key: key of the document to get from the datastore
|
|
1143
1156
|
:return: true/false depending if the document exists or not
|
|
1144
1157
|
"""
|
|
1145
|
-
|
|
1158
|
+
if self.ilm_config:
|
|
1159
|
+
return self._search_exists(key)
|
|
1160
|
+
|
|
1161
|
+
try:
|
|
1162
|
+
return cast(bool, self.with_retries(self.datastore.client.exists, index=self.name, id=key, _source=False))
|
|
1163
|
+
except elasticsearch.BadRequestError:
|
|
1164
|
+
logger.exception(
|
|
1165
|
+
"Single-document existence check failed on %s; falling back to an alias-safe search.",
|
|
1166
|
+
self.name,
|
|
1167
|
+
)
|
|
1168
|
+
return self._search_exists(key)
|
|
1169
|
+
|
|
1170
|
+
def _raise_document_not_found(self, key: str, result: Any) -> typing.NoReturn:
|
|
1171
|
+
"""Raise the same error shape as an Elasticsearch document lookup for an empty search."""
|
|
1172
|
+
meta = ApiResponseMeta(
|
|
1173
|
+
status=404,
|
|
1174
|
+
http_version="1.1",
|
|
1175
|
+
headers=cast(Any, {}),
|
|
1176
|
+
duration=0.0,
|
|
1177
|
+
node=cast(Any, None),
|
|
1178
|
+
)
|
|
1179
|
+
raise elasticsearch.exceptions.NotFoundError(f"Document with id {key} not found", meta, result)
|
|
1146
1180
|
|
|
1147
1181
|
@overload
|
|
1148
1182
|
def _get(self, key, retries, version: Literal[False]) -> dict[str, Any]: ...
|
|
@@ -1154,7 +1188,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
1154
1188
|
"""Versioned get-save for atomic update has two paths:
|
|
1155
1189
|
1. Document doesn't exist at all. Create token will be returned for version.
|
|
1156
1190
|
This way only the first query to try and create the document will succeed.
|
|
1157
|
-
2. Document exists
|
|
1191
|
+
2. Document exists. A version string with the info needed to do a versioned save is returned.
|
|
1158
1192
|
|
|
1159
1193
|
The create token is needed to differentiate between "I'm saving a new
|
|
1160
1194
|
document non-atomic (version=None)" and "I'm saving a new document
|
|
@@ -1173,11 +1207,29 @@ class ESCollection(Generic[ModelType]):
|
|
|
1173
1207
|
done = False
|
|
1174
1208
|
while not done:
|
|
1175
1209
|
try:
|
|
1176
|
-
|
|
1210
|
+
if self.ilm_config:
|
|
1211
|
+
result = self.with_retries(
|
|
1212
|
+
self.datastore.client.search,
|
|
1213
|
+
index=self.name,
|
|
1214
|
+
query={"ids": {"values": [key]}},
|
|
1215
|
+
size=1,
|
|
1216
|
+
seq_no_primary_term=True,
|
|
1217
|
+
)
|
|
1218
|
+
hits = result["hits"]["hits"]
|
|
1219
|
+
if not hits:
|
|
1220
|
+
self._raise_document_not_found(key, result)
|
|
1221
|
+
doc = hits[0]
|
|
1222
|
+
else:
|
|
1223
|
+
doc = self.with_retries(self.datastore.client.get, index=self.name, id=key)
|
|
1224
|
+
|
|
1177
1225
|
if version:
|
|
1226
|
+
if self.ilm_config:
|
|
1227
|
+
version_token = f"{doc['_index']}---{doc['_seq_no']}---{doc['_primary_term']}"
|
|
1228
|
+
else:
|
|
1229
|
+
version_token = f"{doc['_seq_no']}---{doc['_primary_term']}"
|
|
1178
1230
|
return (
|
|
1179
1231
|
normalize_output(doc["_source"]),
|
|
1180
|
-
|
|
1232
|
+
version_token,
|
|
1181
1233
|
)
|
|
1182
1234
|
return normalize_output(doc["_source"])
|
|
1183
1235
|
except elasticsearch.exceptions.NotFoundError:
|
|
@@ -1196,6 +1248,19 @@ class ESCollection(Generic[ModelType]):
|
|
|
1196
1248
|
|
|
1197
1249
|
return None
|
|
1198
1250
|
|
|
1251
|
+
def _get_version_write_target(self, version: str) -> tuple[str, str, str]:
|
|
1252
|
+
"""Return the concrete write target and optimistic-concurrency values from a version token."""
|
|
1253
|
+
version_parts = version.split("---")
|
|
1254
|
+
if len(version_parts) == 3:
|
|
1255
|
+
index, seq_no, primary_term = version_parts
|
|
1256
|
+
return index, seq_no, primary_term
|
|
1257
|
+
|
|
1258
|
+
if len(version_parts) == 2:
|
|
1259
|
+
seq_no, primary_term = version_parts
|
|
1260
|
+
return self.name, seq_no, primary_term
|
|
1261
|
+
|
|
1262
|
+
raise DataStoreException(f"Invalid version token for {self.name}: {version!r}")
|
|
1263
|
+
|
|
1199
1264
|
@overload
|
|
1200
1265
|
def get(self, key, as_obj: Literal[True], version: Literal[True]) -> tuple[ModelType | None, str]: ...
|
|
1201
1266
|
|
|
@@ -1325,13 +1390,17 @@ class ESCollection(Generic[ModelType]):
|
|
|
1325
1390
|
|
|
1326
1391
|
saved_data["id"] = key
|
|
1327
1392
|
operation = "index"
|
|
1393
|
+
index = self.name
|
|
1328
1394
|
seq_no = None
|
|
1329
1395
|
primary_term = None
|
|
1330
1396
|
|
|
1397
|
+
if version is None and self.ilm_config:
|
|
1398
|
+
_, version = self.get_if_exists(key, as_obj=False, version=True)
|
|
1399
|
+
|
|
1331
1400
|
if version == CREATE_TOKEN:
|
|
1332
1401
|
operation = "create"
|
|
1333
1402
|
elif version:
|
|
1334
|
-
seq_no, primary_term =
|
|
1403
|
+
index, seq_no, primary_term = self._get_version_write_target(version)
|
|
1335
1404
|
|
|
1336
1405
|
if refresh == "true":
|
|
1337
1406
|
logger.warning(
|
|
@@ -1341,7 +1410,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
1341
1410
|
try:
|
|
1342
1411
|
self.with_retries(
|
|
1343
1412
|
self.datastore.client.index,
|
|
1344
|
-
index=
|
|
1413
|
+
index=index,
|
|
1345
1414
|
id=key,
|
|
1346
1415
|
document=json.dumps(saved_data),
|
|
1347
1416
|
op_type=operation,
|
|
@@ -1541,15 +1610,20 @@ class ESCollection(Generic[ModelType]):
|
|
|
1541
1610
|
"""
|
|
1542
1611
|
operations = self._validate_operations(operations)
|
|
1543
1612
|
script = self._create_scripts_from_operations(operations)
|
|
1613
|
+
index = self.name
|
|
1544
1614
|
seq_no = None
|
|
1545
1615
|
primary_term = None
|
|
1616
|
+
|
|
1617
|
+
if version is None and self.ilm_config:
|
|
1618
|
+
_, version = self.get_if_exists(key, as_obj=False, version=True)
|
|
1619
|
+
|
|
1546
1620
|
if version:
|
|
1547
|
-
seq_no, primary_term =
|
|
1621
|
+
index, seq_no, primary_term = self._get_version_write_target(version)
|
|
1548
1622
|
|
|
1549
1623
|
try:
|
|
1550
1624
|
res = self.with_retries(
|
|
1551
1625
|
self.datastore.client.update,
|
|
1552
|
-
index=
|
|
1626
|
+
index=index,
|
|
1553
1627
|
id=key,
|
|
1554
1628
|
script=script,
|
|
1555
1629
|
if_seq_no=seq_no,
|
|
@@ -1559,7 +1633,11 @@ class ESCollection(Generic[ModelType]):
|
|
|
1559
1633
|
)
|
|
1560
1634
|
return (
|
|
1561
1635
|
res["result"] == "updated",
|
|
1562
|
-
|
|
1636
|
+
(
|
|
1637
|
+
f"{res['_index']}---{res['_seq_no']}---{res['_primary_term']}"
|
|
1638
|
+
if self.ilm_config
|
|
1639
|
+
else f"{res['_seq_no']}---{res['_primary_term']}"
|
|
1640
|
+
),
|
|
1563
1641
|
)
|
|
1564
1642
|
except elasticsearch.NotFoundError as e:
|
|
1565
1643
|
logger.warning("Update - elasticsearch.NotFoundError: %s %s", e.message, e.info)
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Find and optionally remove duplicate documents from an ILM collection."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import heapq
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def iter_index_ids(collection: Any, index: str, batch_size: int) -> Iterator[str]:
|
|
11
|
+
"""Yield document IDs from *index* in sorted order without retaining prior pages."""
|
|
12
|
+
search_after = None
|
|
13
|
+
|
|
14
|
+
while True:
|
|
15
|
+
search_args = {
|
|
16
|
+
"index": index,
|
|
17
|
+
"query": {"match_all": {}},
|
|
18
|
+
"sort": [{"id": "asc"}],
|
|
19
|
+
"size": batch_size,
|
|
20
|
+
"_source": False,
|
|
21
|
+
"track_total_hits": False,
|
|
22
|
+
}
|
|
23
|
+
if search_after is not None:
|
|
24
|
+
search_args["search_after"] = search_after
|
|
25
|
+
|
|
26
|
+
response = collection.with_retries(collection.datastore.client.search, **search_args)
|
|
27
|
+
hits = response["hits"]["hits"]
|
|
28
|
+
if not hits:
|
|
29
|
+
return
|
|
30
|
+
|
|
31
|
+
for hit in hits:
|
|
32
|
+
yield hit["_id"]
|
|
33
|
+
|
|
34
|
+
search_after = hits[-1]["sort"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _push_next_id(heap: list[tuple[str, str, Iterator[str]]], index: str, document_ids: Iterator[str]) -> None:
|
|
38
|
+
try:
|
|
39
|
+
heapq.heappush(heap, (next(document_ids), index, document_ids))
|
|
40
|
+
except StopIteration:
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def iter_duplicates(collection: Any, indexes: list[str], batch_size: int) -> Iterator[tuple[str, list[str]]]:
|
|
45
|
+
"""Yield each duplicated ID and the physical indexes that contain it.
|
|
46
|
+
|
|
47
|
+
This is a k-way merge over paginated, sorted searches. Its memory use is bounded by
|
|
48
|
+
the number of physical indexes and one Elasticsearch result page per active iterator.
|
|
49
|
+
"""
|
|
50
|
+
heap: list[tuple[str, str, Iterator[str]]] = []
|
|
51
|
+
for index in indexes:
|
|
52
|
+
document_ids = iter_index_ids(collection, index, batch_size)
|
|
53
|
+
_push_next_id(heap, index, document_ids)
|
|
54
|
+
|
|
55
|
+
while heap:
|
|
56
|
+
document_id, index, document_ids = heapq.heappop(heap)
|
|
57
|
+
duplicate_indexes = [index]
|
|
58
|
+
_push_next_id(heap, index, document_ids)
|
|
59
|
+
|
|
60
|
+
while heap and heap[0][0] == document_id:
|
|
61
|
+
_, index, document_ids = heapq.heappop(heap)
|
|
62
|
+
duplicate_indexes.append(index)
|
|
63
|
+
_push_next_id(heap, index, document_ids)
|
|
64
|
+
|
|
65
|
+
if len(duplicate_indexes) > 1:
|
|
66
|
+
yield document_id, duplicate_indexes
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def delete_duplicates(collection: Any, document_id: str, indexes: list[str]) -> int:
|
|
70
|
+
"""Delete all supplied physical-index copies of a document ID."""
|
|
71
|
+
deleted = 0
|
|
72
|
+
for index in indexes:
|
|
73
|
+
response = collection.with_retries(collection.datastore.client.delete, index=index, id=document_id)
|
|
74
|
+
if response["result"] == "deleted":
|
|
75
|
+
deleted += 1
|
|
76
|
+
return deleted
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def main() -> int:
|
|
80
|
+
"""Run the interactive duplicate scan for an ILM collection."""
|
|
81
|
+
parser = argparse.ArgumentParser(
|
|
82
|
+
description="Find duplicate document IDs across an ILM collection's physical indexes."
|
|
83
|
+
)
|
|
84
|
+
parser.add_argument("collection", help="Collection name registered in the datastore.")
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"--batch-size",
|
|
87
|
+
type=int,
|
|
88
|
+
default=1_000,
|
|
89
|
+
help="IDs fetched from each physical index per Elasticsearch request (default: 1000).",
|
|
90
|
+
)
|
|
91
|
+
args = parser.parse_args()
|
|
92
|
+
|
|
93
|
+
if args.batch_size <= 0:
|
|
94
|
+
parser.error("--batch-size must be a positive integer.")
|
|
95
|
+
|
|
96
|
+
from howler.datastore.collection import ESCollection
|
|
97
|
+
|
|
98
|
+
# Checking duplicates must not create collections or alter their ILM configuration.
|
|
99
|
+
ESCollection.IGNORE_ENSURE_COLLECTION = True
|
|
100
|
+
ESCollection.ENSURE_COLLECTION_WARNED = True
|
|
101
|
+
|
|
102
|
+
from howler.common import loader
|
|
103
|
+
|
|
104
|
+
datastore = loader.datastore(archive_access=False)
|
|
105
|
+
collection_names = datastore.ds.get_models()
|
|
106
|
+
if args.collection not in collection_names:
|
|
107
|
+
parser.error(
|
|
108
|
+
f"Unknown collection '{args.collection}'. Valid collections: {', '.join(sorted(collection_names))}"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
collection = getattr(datastore, args.collection)
|
|
112
|
+
if collection.ilm_config is None:
|
|
113
|
+
print(f"Collection '{args.collection}' is not ILM-enabled; nothing to check.")
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
collection._refresh_ilm_index_name()
|
|
117
|
+
indexes = collection.index_list_full
|
|
118
|
+
if len(indexes) < 2:
|
|
119
|
+
print(f"Collection '{args.collection}' has fewer than two physical ILM indexes; nothing to check.")
|
|
120
|
+
return 0
|
|
121
|
+
|
|
122
|
+
print(f"Scanning for duplicate IDs in: {', '.join(indexes)}")
|
|
123
|
+
duplicates_found = 0
|
|
124
|
+
copies_deleted = 0
|
|
125
|
+
index_order = {index: position for position, index in enumerate(indexes)}
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
for document_id, duplicate_indexes in iter_duplicates(collection, indexes, args.batch_size):
|
|
129
|
+
duplicates_found += 1
|
|
130
|
+
keep_index = min(duplicate_indexes, key=index_order.__getitem__)
|
|
131
|
+
stale_indexes = [index for index in duplicate_indexes if index != keep_index]
|
|
132
|
+
print(
|
|
133
|
+
f"Duplicate ID '{document_id}' found in {', '.join(duplicate_indexes)}. "
|
|
134
|
+
f"Keeping the newest copy in '{keep_index}'."
|
|
135
|
+
)
|
|
136
|
+
answer = input(f"Delete copies from {', '.join(stale_indexes)}? [yes/NO] ")
|
|
137
|
+
if answer.lower() in {"y", "yes"}:
|
|
138
|
+
deleted_count = delete_duplicates(collection, document_id, stale_indexes)
|
|
139
|
+
copies_deleted += deleted_count
|
|
140
|
+
print(f"Deleted {deleted_count} copy/copies of '{document_id}'.")
|
|
141
|
+
except KeyboardInterrupt:
|
|
142
|
+
print("\nInterrupted; all completed deletions have been retained.", file=sys.stderr)
|
|
143
|
+
return 130
|
|
144
|
+
finally:
|
|
145
|
+
datastore.ds.close()
|
|
146
|
+
|
|
147
|
+
print(f"Scan complete. Found {duplicates_found} duplicate ID(s); deleted {copies_deleted} stale copy/copies.")
|
|
148
|
+
return 0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
if __name__ == "__main__":
|
|
152
|
+
sys.exit(main())
|
|
@@ -15,6 +15,7 @@ from howler.common.exceptions import HowlerRuntimeError
|
|
|
15
15
|
from howler.common.loader import datastore
|
|
16
16
|
from howler.datastore.collection import ESCollection
|
|
17
17
|
from howler.odm.base import Model
|
|
18
|
+
from howler.utils.compat import Self
|
|
18
19
|
|
|
19
20
|
ModelType = TypeVar("ModelType", bound=Model)
|
|
20
21
|
|
|
@@ -28,10 +29,10 @@ class _ObjectsDescriptor(Generic[ModelType]):
|
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
31
|
@overload
|
|
31
|
-
def __get__(self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
32
|
+
def __get__(self: Self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
32
33
|
|
|
33
34
|
@overload
|
|
34
|
-
def __get__(self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
35
|
+
def __get__(self: Self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
35
36
|
|
|
36
37
|
def __get__(self, obj: ModelType | None, objtype: type[ModelType] | None = None) -> ESCollection[ModelType]:
|
|
37
38
|
"""Return the ESCollection for the owner class.
|
|
@@ -82,7 +83,11 @@ class DatastoreMixin(Generic[ModelType]):
|
|
|
82
83
|
"""
|
|
83
84
|
return datastore()
|
|
84
85
|
|
|
85
|
-
def save(
|
|
86
|
+
def save(
|
|
87
|
+
self: Self,
|
|
88
|
+
refresh: Literal["true", "false", "wait_for"] | None = None,
|
|
89
|
+
version: str | None = None,
|
|
90
|
+
) -> bool:
|
|
86
91
|
"""Persist the current model instance to the datastore.
|
|
87
92
|
|
|
88
93
|
Determines the target index from the lowercase class name, extracts the
|
|
@@ -96,5 +101,6 @@ class DatastoreMixin(Generic[ModelType]):
|
|
|
96
101
|
index_name = self.__class__.__name__.lower()
|
|
97
102
|
id_field = self.__class__._Model__id_field # type: ignore[attr-defined]
|
|
98
103
|
current_id = attrgetter(id_field)(self)
|
|
104
|
+
collection = self.ds[index_name]
|
|
99
105
|
|
|
100
|
-
return
|
|
106
|
+
return collection.save(current_id, self, version=version, refresh=refresh)
|