howler-api 4.0.0.dev973__tar.gz → 4.0.0.dev982__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/PKG-INFO +1 -1
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/demote.py +2 -1
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/promote.py +6 -1
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/collection.py +244 -61
- howler_api-4.0.0.dev982/howler/external/reindex_data.py +134 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/hit.py +10 -1
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/helper.py +1 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/howler_data.py +6 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/random_data.py +1 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/hit_service.py +1 -1
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/pyproject.toml +1 -1
- howler_api-4.0.0.dev973/howler/external/reindex_data.py +0 -64
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/add_label.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/add_to_bundle.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/change_field.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/example_plugin.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/prioritization.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/remove_from_bundle.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/remove_label.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/transition.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/base.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/socket.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/action.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/analytic.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/auth.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/clue.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/configs.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/dossier.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/help.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/hit.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/notebook.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/overview.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/search.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/template.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/tool.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/user.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/utils/etag.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/view.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/app.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/classification.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/classification.yml +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/exceptions.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/loader.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/audit.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/format.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/net.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/net_static.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/random_user.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/swagger.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/config.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/action_queue_worker.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/retention.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/rules.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/view_cleanup.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/bulk.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/constants.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/exceptions.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/howler_store.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/migrations/fix_process.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/operations.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/schemas.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/store.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/build.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/schemas.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/types.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/error.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_mitre.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_sigma_rules.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_tlds.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/wipe_databases.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/gunicorn_config.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/healthz.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/azure.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/discover.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/oauth.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/search.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/workflow.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/ws.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/base.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/charter.txt +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/howler_enum.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/action.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/analytic.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/assemblyline.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/aws.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/azure.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/cbs.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/clue.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/config.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/dossier.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/agent.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/autonomous_system.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/client.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/cloud.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/code_signature.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/container.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/dns.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/egress.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/elf.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/email.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/error.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/event.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/faas.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/file.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/geo.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/group.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/hash.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/host.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/http.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/ingress.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/interface.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/network.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/observer.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/organization.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/os.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/pe.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/process.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/registry.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/related.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/rule.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/server.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/threat.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/tls.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/url.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/user.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/user_agent.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/vulnerability.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/gcp.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/hit.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/lead.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/localized_label.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/overview.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/pivot.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/template.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/user.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/view.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/randomizer.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/patched.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/plugins/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/plugins/config.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/README.md +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/counters.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/events.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/hash.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/lock.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/comms.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/multi.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/named.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/priority.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/set.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/user_quota_tracker.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/socket.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/utils.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/action_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/analytic_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/auth_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/config_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/dossier_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/event_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/jwt_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/lucene_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/notebook_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/overview_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/template_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/user_service.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/telemetry.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/annotations.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/chunk.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/compat.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/constants.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/dict_utils.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/isotime.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/list_utils.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/lucene.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/path.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/socket_utils.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/str_utils.py +0 -0
- {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/uid.py +0 -0
|
@@ -76,6 +76,7 @@ def execute(
|
|
|
76
76
|
*hit_helper.demote_hit(escalation=escalation),
|
|
77
77
|
odm_helper.update("howler.assessment", None),
|
|
78
78
|
odm_helper.update("howler.rationale", None),
|
|
79
|
+
odm_helper.update("howler.assignment", None),
|
|
79
80
|
],
|
|
80
81
|
)
|
|
81
82
|
else:
|
|
@@ -93,7 +94,7 @@ def execute(
|
|
|
93
94
|
ds.hit.update_by_query(
|
|
94
95
|
query,
|
|
95
96
|
[
|
|
96
|
-
*hit_helper.assess_hit(assessment, rationale),
|
|
97
|
+
*hit_helper.assess_hit(assessment, rationale, user=(user if user else "automation")),
|
|
97
98
|
odm_helper.update(
|
|
98
99
|
"howler.assignment",
|
|
99
100
|
user.get("uname", "automation") if user else "automation",
|
|
@@ -10,6 +10,7 @@ from howler.odm.models.howler_data import (
|
|
|
10
10
|
AssessmentEscalationMap,
|
|
11
11
|
Escalation,
|
|
12
12
|
)
|
|
13
|
+
from howler.odm.models.user import User
|
|
13
14
|
from howler.utils.str_utils import sanitize_lucene_query
|
|
14
15
|
|
|
15
16
|
OPERATION_ID = "promote"
|
|
@@ -26,6 +27,7 @@ def execute(
|
|
|
26
27
|
escalation: Escalation = Escalation.ALERT,
|
|
27
28
|
assessment: Optional[str] = None,
|
|
28
29
|
rationale: Optional[str] = None,
|
|
30
|
+
user: Optional[User] = None,
|
|
29
31
|
**kwargs,
|
|
30
32
|
):
|
|
31
33
|
"""Promote a hit.
|
|
@@ -73,6 +75,7 @@ def execute(
|
|
|
73
75
|
*hit_helper.promote_hit(escalation=escalation),
|
|
74
76
|
odm_helper.update("howler.assessment", None),
|
|
75
77
|
odm_helper.update("howler.rationale", None),
|
|
78
|
+
odm_helper.update("howler.assignment", None),
|
|
76
79
|
],
|
|
77
80
|
)
|
|
78
81
|
else:
|
|
@@ -87,7 +90,9 @@ def execute(
|
|
|
87
90
|
)
|
|
88
91
|
return report
|
|
89
92
|
|
|
90
|
-
ds.hit.update_by_query(
|
|
93
|
+
ds.hit.update_by_query(
|
|
94
|
+
query, hit_helper.assess_hit(assessment, rationale, user=(user if user else "automation"))
|
|
95
|
+
)
|
|
91
96
|
|
|
92
97
|
report.append(
|
|
93
98
|
{
|
|
@@ -480,8 +480,8 @@ class ESCollection(Generic[ModelType]):
|
|
|
480
480
|
else:
|
|
481
481
|
raise
|
|
482
482
|
|
|
483
|
-
def _safe_index_copy(self, copy_function, src, target, settings=None, min_status="yellow"):
|
|
484
|
-
options_client = self.datastore.client.options(request_timeout=
|
|
483
|
+
def _safe_index_copy(self, copy_function, src, target, settings=None, min_status="yellow", request_timeout=60):
|
|
484
|
+
options_client = self.datastore.client.options(request_timeout=request_timeout)
|
|
485
485
|
timed_function = getattr(options_client.indices, copy_function.__name__)
|
|
486
486
|
ret = timed_function(index=src, target=target, settings=settings)
|
|
487
487
|
if not ret["acknowledged"]:
|
|
@@ -703,22 +703,80 @@ class ESCollection(Generic[ModelType]):
|
|
|
703
703
|
settings=clone_finish_settings,
|
|
704
704
|
)
|
|
705
705
|
|
|
706
|
-
def
|
|
707
|
-
"""
|
|
708
|
-
specified in self.datastore.hosts.
|
|
706
|
+
def _index_doc_count(self, index: str) -> int:
|
|
707
|
+
"""Return the number of documents in a physical index.
|
|
709
708
|
|
|
710
|
-
:
|
|
709
|
+
:param index: the name of the physical index to count documents in
|
|
710
|
+
:return: the number of documents currently stored in the index
|
|
711
|
+
"""
|
|
712
|
+
self.with_retries(self.datastore.client.indices.refresh, index=index)
|
|
713
|
+
return self.with_retries(self.datastore.client.count, index=index)["count"]
|
|
714
|
+
|
|
715
|
+
def reindex(self, allow_failures: bool = False, request_timeout: int = 60):
|
|
716
|
+
"""Reindex all the data of the collection into a freshly mapped index.
|
|
717
|
+
|
|
718
|
+
For every index in ``self.index_list`` the data is copied into a temporary
|
|
719
|
+
``__reindex`` index that uses the current mappings/settings. Writes to the source
|
|
720
|
+
index are blocked while the copy runs so the reindex result and document counts can
|
|
721
|
+
be validated before the original index is deleted. The temporary index is only
|
|
722
|
+
collapsed back onto the original name once those checks pass.
|
|
723
|
+
|
|
724
|
+
:param allow_failures: when ``True``, proceed even if the reindex reported document
|
|
725
|
+
failures or version conflicts, or if the document counts do not match. This is
|
|
726
|
+
DESTRUCTIVE: documents that could not be converted to the new mappings will be
|
|
727
|
+
permanently dropped. Only use this for intentional lossy migrations.
|
|
728
|
+
:param request_timeout: transport timeout in seconds for synchronous index-copy
|
|
729
|
+
operations. Defaults to 60 seconds.
|
|
730
|
+
:return: ``True`` when the reindex (and validation) completed successfully on all
|
|
731
|
+
indexes.
|
|
732
|
+
:raises DataStoreException: if a reindex reported failures/conflicts, or if the
|
|
733
|
+
document count of the reindexed data does not match the source, and
|
|
734
|
+
``allow_failures`` is ``False``. The ``__reindex`` index is left in place so the
|
|
735
|
+
operation can be recovered with :meth:`reindex_cleanup`.
|
|
711
736
|
"""
|
|
712
737
|
logger.warning("Beginning Reindex")
|
|
713
738
|
for index in self.index_list:
|
|
714
739
|
new_name = f"{index}__reindex"
|
|
715
740
|
index_data = None
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
741
|
+
source_count = None
|
|
742
|
+
source_writes_blocked = False
|
|
743
|
+
|
|
744
|
+
source_exists = self.with_retries(self.datastore.client.indices.exists, index=index)
|
|
745
|
+
target_exists = self.with_retries(self.datastore.client.indices.exists, index=new_name)
|
|
746
|
+
|
|
747
|
+
# Never reindex while a '__reindex' index already exists. Its presence means a previous
|
|
748
|
+
# reindex was interrupted, and the leftover may be stale or incomplete. Committing it
|
|
749
|
+
# could silently replace live data, so force the operator to reconcile the state with
|
|
750
|
+
# --cleanup before any new reindex is attempted.
|
|
751
|
+
if target_exists:
|
|
752
|
+
raise DataStoreException(
|
|
753
|
+
f"A leftover reindex index '{new_name}' already exists. This usually means a previous "
|
|
754
|
+
f"reindex was interrupted. Refusing to reindex because '{new_name}' may contain stale "
|
|
755
|
+
f"or incomplete data. Run the reindex script with --cleanup to reconcile the state "
|
|
756
|
+
f"if '{index}' still exists, or manually recover '{new_name}' if the source index is "
|
|
757
|
+
f"missing, then retry the reindex."
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
if not source_exists:
|
|
761
|
+
logger.warning("Neither %s nor %s exist, nothing to reindex.", index, new_name)
|
|
762
|
+
continue
|
|
763
|
+
|
|
764
|
+
try:
|
|
719
765
|
# Get information about the index to reindex
|
|
720
766
|
index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
|
|
721
767
|
|
|
768
|
+
logger.warning("Block writes to source index %s", index)
|
|
769
|
+
self.with_retries(
|
|
770
|
+
self.datastore.client.indices.put_settings,
|
|
771
|
+
index=index,
|
|
772
|
+
settings=write_block_settings,
|
|
773
|
+
)
|
|
774
|
+
source_writes_blocked = True
|
|
775
|
+
|
|
776
|
+
# Record the number of documents we expect to migrate after writes are blocked.
|
|
777
|
+
source_count = self._index_doc_count(index)
|
|
778
|
+
logger.warning("Source index %s contains %s document(s)", index, source_count)
|
|
779
|
+
|
|
722
780
|
# Create reindex target
|
|
723
781
|
logger.warning("Creating new index with name %s", new_name)
|
|
724
782
|
self.with_retries(
|
|
@@ -728,32 +786,6 @@ class ESCollection(Generic[ModelType]):
|
|
|
728
786
|
settings=self._get_index_settings(),
|
|
729
787
|
)
|
|
730
788
|
|
|
731
|
-
# For all aliases related to the index, add a new alias to the reindex index
|
|
732
|
-
for alias, alias_data in index_data["aliases"].items():
|
|
733
|
-
# Make the reindex index the new write index if the original index was
|
|
734
|
-
if alias_data.get("is_write_index", True):
|
|
735
|
-
alias_actions = [
|
|
736
|
-
{
|
|
737
|
-
"add": {
|
|
738
|
-
"index": new_name,
|
|
739
|
-
"alias": alias,
|
|
740
|
-
"is_write_index": True,
|
|
741
|
-
}
|
|
742
|
-
},
|
|
743
|
-
{
|
|
744
|
-
"add": {
|
|
745
|
-
"index": index,
|
|
746
|
-
"alias": alias,
|
|
747
|
-
"is_write_index": False,
|
|
748
|
-
}
|
|
749
|
-
},
|
|
750
|
-
]
|
|
751
|
-
else:
|
|
752
|
-
alias_actions = [{"add": {"index": new_name, "alias": alias}}]
|
|
753
|
-
|
|
754
|
-
logger.warning("Updating alias %s", alias)
|
|
755
|
-
self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
|
|
756
|
-
|
|
757
789
|
# Reindex data into target
|
|
758
790
|
logger.warning("Beginning reindex from %s to %s", index, new_name)
|
|
759
791
|
r_task = self.with_retries(
|
|
@@ -763,23 +795,29 @@ class ESCollection(Generic[ModelType]):
|
|
|
763
795
|
wait_for_completion=False,
|
|
764
796
|
)
|
|
765
797
|
logger.warning("Reindex taskId: %s", r_task["task"])
|
|
766
|
-
self._get_task_results(r_task)
|
|
798
|
+
reindex_result = self._get_task_results(r_task)
|
|
767
799
|
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
800
|
+
# Validate the reindex did not silently drop or conflict on documents before
|
|
801
|
+
# we commit to deleting the source index further down.
|
|
802
|
+
self._validate_reindex_result(index, new_name, reindex_result, allow_failures)
|
|
771
803
|
|
|
772
804
|
logger.warning("Committing reindexed data in index %s", new_name)
|
|
773
805
|
self.with_retries(self.datastore.client.indices.refresh, index=new_name)
|
|
774
806
|
self.with_retries(self.datastore.client.indices.clear_cache, index=new_name)
|
|
775
807
|
|
|
808
|
+
# Compare the document counts of the source and reindexed indexes before deleting
|
|
809
|
+
# the source so a silent document drop is caught.
|
|
810
|
+
target_count = self._index_doc_count(new_name)
|
|
811
|
+
self._validate_reindex_counts(index, new_name, source_count, target_count, allow_failures)
|
|
812
|
+
|
|
776
813
|
logger.warning("Deleting old index %s", index)
|
|
777
|
-
|
|
778
|
-
|
|
814
|
+
self.with_retries(self.datastore.client.indices.delete, index=index)
|
|
815
|
+
source_writes_blocked = False
|
|
779
816
|
|
|
780
|
-
logger.warning("Block
|
|
817
|
+
logger.warning("Block writes to reindex target %s", new_name)
|
|
781
818
|
self.with_retries(
|
|
782
819
|
self.datastore.client.indices.put_settings,
|
|
820
|
+
index=new_name,
|
|
783
821
|
settings=write_block_settings,
|
|
784
822
|
)
|
|
785
823
|
|
|
@@ -790,36 +828,181 @@ class ESCollection(Generic[ModelType]):
|
|
|
790
828
|
new_name,
|
|
791
829
|
index,
|
|
792
830
|
settings=self._get_index_settings(),
|
|
831
|
+
request_timeout=request_timeout,
|
|
793
832
|
)
|
|
794
833
|
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
}
|
|
806
|
-
},
|
|
807
|
-
{"remove_index": {"index": new_name}},
|
|
808
|
-
]
|
|
809
|
-
self.with_retries(
|
|
810
|
-
self.datastore.client.indices.update_aliases,
|
|
811
|
-
actions=alias_actions,
|
|
812
|
-
)
|
|
834
|
+
alias_actions = []
|
|
835
|
+
aliases = index_data.get("aliases", {}) or {self.name: {"is_write_index": True}}
|
|
836
|
+
for alias, alias_data in aliases.items():
|
|
837
|
+
alias_action = {"index": index, "alias": alias}
|
|
838
|
+
alias_action.update(alias_data)
|
|
839
|
+
if alias == self.name or alias_data.get("is_write_index", False):
|
|
840
|
+
alias_action["is_write_index"] = True
|
|
841
|
+
alias_actions.append({"add": alias_action})
|
|
842
|
+
alias_actions.append({"remove_index": {"index": new_name}})
|
|
843
|
+
self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
|
|
813
844
|
|
|
814
845
|
if self.with_retries(self.datastore.client.indices.exists, index=new_name):
|
|
815
846
|
logger.warning("Deleting reindex target %s", new_name)
|
|
816
847
|
self.with_retries(self.datastore.client.indices.delete, index=new_name)
|
|
817
848
|
finally:
|
|
818
|
-
|
|
849
|
+
if self.with_retries(self.datastore.client.indices.exists, index=index):
|
|
850
|
+
logger.warning("Unblock writes to the index")
|
|
851
|
+
self.with_retries(
|
|
852
|
+
self.datastore.client.indices.put_settings,
|
|
853
|
+
index=index,
|
|
854
|
+
settings=write_unblock_settings,
|
|
855
|
+
)
|
|
856
|
+
except Exception:
|
|
857
|
+
if source_writes_blocked and self.with_retries(self.datastore.client.indices.exists, index=index):
|
|
858
|
+
logger.warning("Unblock writes to source index %s after failed reindex", index)
|
|
819
859
|
self.with_retries(
|
|
820
860
|
self.datastore.client.indices.put_settings,
|
|
861
|
+
index=index,
|
|
821
862
|
settings=write_unblock_settings,
|
|
822
863
|
)
|
|
864
|
+
raise
|
|
865
|
+
|
|
866
|
+
return True
|
|
867
|
+
|
|
868
|
+
def _validate_reindex_result(self, index, new_name, reindex_result, allow_failures):
|
|
869
|
+
"""Validate the result of an Elasticsearch reindex task.
|
|
870
|
+
|
|
871
|
+
:param index: the source index being reindexed
|
|
872
|
+
:param new_name: the temporary ``__reindex`` index being written to
|
|
873
|
+
:param reindex_result: the ``response``/``status`` payload returned by the reindex task
|
|
874
|
+
:param allow_failures: when ``True``, log the problems but do not abort
|
|
875
|
+
:raises DataStoreException: if the reindex reported failures or version conflicts and
|
|
876
|
+
``allow_failures`` is ``False``
|
|
877
|
+
"""
|
|
878
|
+
failures = reindex_result.get("failures", []) if reindex_result else []
|
|
879
|
+
version_conflicts = reindex_result.get("version_conflicts", 0) if reindex_result else 0
|
|
880
|
+
|
|
881
|
+
if not failures and not version_conflicts:
|
|
882
|
+
return
|
|
883
|
+
|
|
884
|
+
# Summarize the failures so the operator understands what went wrong
|
|
885
|
+
summary = (
|
|
886
|
+
f"Reindex of {index} into {new_name} reported {len(failures)} document failure(s) "
|
|
887
|
+
f"and {version_conflicts} version conflict(s)."
|
|
888
|
+
)
|
|
889
|
+
for failure in failures[:10]:
|
|
890
|
+
cause = failure.get("cause", failure)
|
|
891
|
+
logger.error(
|
|
892
|
+
"Reindex failure on document %s: %s - %s",
|
|
893
|
+
failure.get("id", "<unknown>"),
|
|
894
|
+
cause.get("type", "<unknown>"),
|
|
895
|
+
cause.get("reason", cause),
|
|
896
|
+
)
|
|
897
|
+
if len(failures) > 10:
|
|
898
|
+
logger.error("... and %s additional failure(s) not shown.", len(failures) - 10)
|
|
899
|
+
|
|
900
|
+
if allow_failures:
|
|
901
|
+
logger.warning("%s Proceeding anyway because allow_failures is set (DESTRUCTIVE).", summary)
|
|
902
|
+
return
|
|
903
|
+
|
|
904
|
+
raise DataStoreException(
|
|
905
|
+
f"{summary} Aborting before deleting the source index to prevent data loss. The '{new_name}' "
|
|
906
|
+
f"index has been left in place; run the reindex script with --cleanup to remove it and restore "
|
|
907
|
+
f"'{index}', then retry. Re-run with --allow-failures only if you intend to drop the "
|
|
908
|
+
f"un-convertible documents."
|
|
909
|
+
)
|
|
910
|
+
|
|
911
|
+
def _validate_reindex_counts(self, index, new_name, source_count, target_count, allow_failures):
|
|
912
|
+
"""Validate that the reindexed index contains the same number of documents as the source.
|
|
913
|
+
|
|
914
|
+
:param index: the source index being reindexed
|
|
915
|
+
:param new_name: the temporary ``__reindex`` index being written to
|
|
916
|
+
:param source_count: the number of documents in the source index
|
|
917
|
+
:param target_count: the number of documents in the reindexed index
|
|
918
|
+
:param allow_failures: when ``True``, log the mismatch but do not abort
|
|
919
|
+
:raises DataStoreException: if the counts differ and ``allow_failures`` is ``False``
|
|
920
|
+
"""
|
|
921
|
+
if source_count == target_count:
|
|
922
|
+
logger.warning("Document count validated: %s document(s) in both %s and %s", source_count, index, new_name)
|
|
923
|
+
return
|
|
924
|
+
|
|
925
|
+
summary = (
|
|
926
|
+
f"Document count mismatch reindexing {index} into {new_name}: "
|
|
927
|
+
f"source has {source_count} document(s) but reindex target has {target_count}."
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
if allow_failures:
|
|
931
|
+
logger.warning("%s Proceeding anyway because allow_failures is set (DESTRUCTIVE).", summary)
|
|
932
|
+
return
|
|
933
|
+
|
|
934
|
+
raise DataStoreException(
|
|
935
|
+
f"{summary} Aborting before deleting the source index to prevent data loss. The '{new_name}' "
|
|
936
|
+
f"index has been left in place; run the reindex script with --cleanup to remove it and restore "
|
|
937
|
+
f"'{index}', then retry. Re-run with --allow-failures only if you intend to accept the count "
|
|
938
|
+
f"mismatch."
|
|
939
|
+
)
|
|
940
|
+
|
|
941
|
+
def reindex_cleanup(self):
|
|
942
|
+
"""Recover from a failed or interrupted :meth:`reindex` run.
|
|
943
|
+
|
|
944
|
+
For every index that still has a leftover ``__reindex`` index, restore the source
|
|
945
|
+
index as the writable target and delete the orphaned ``__reindex`` index. If the
|
|
946
|
+
source index is missing, raise an error instead of deleting ``__reindex`` because
|
|
947
|
+
it may be the only remaining copy of the data.
|
|
948
|
+
|
|
949
|
+
Write blocks set during reindexing are cleared so the collection is usable again.
|
|
950
|
+
|
|
951
|
+
:return: ``True`` when cleanup completed on all indexes
|
|
952
|
+
"""
|
|
953
|
+
logger.warning("Beginning reindex cleanup")
|
|
954
|
+
for index in self.index_list:
|
|
955
|
+
new_name = f"{index}__reindex"
|
|
956
|
+
|
|
957
|
+
if not self.with_retries(self.datastore.client.indices.exists, index=new_name):
|
|
958
|
+
logger.info("No leftover reindex index found for %s, nothing to clean up.", index)
|
|
959
|
+
continue
|
|
960
|
+
|
|
961
|
+
if not self.with_retries(self.datastore.client.indices.exists, index=index):
|
|
962
|
+
raise DataStoreException(
|
|
963
|
+
f"Source index '{index}' is missing but leftover reindex index '{new_name}' exists. "
|
|
964
|
+
f"Cannot safely clean up because '{new_name}' may be the only remaining copy of the data. "
|
|
965
|
+
f"Manually recover '{new_name}' or delete it if the data is no longer needed."
|
|
966
|
+
)
|
|
967
|
+
|
|
968
|
+
logger.warning("Restoring aliases for %s and removing leftover index %s", index, new_name)
|
|
969
|
+
index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
|
|
970
|
+
new_index_data = self.with_retries(self.datastore.client.indices.get, index=new_name)[new_name]
|
|
971
|
+
source_aliases = index_data.get("aliases", {})
|
|
972
|
+
reindex_aliases = new_index_data.get("aliases", {})
|
|
973
|
+
|
|
974
|
+
alias_actions = []
|
|
975
|
+
for alias in sorted(set(source_aliases) | set(reindex_aliases) | {self.name}):
|
|
976
|
+
source_alias_data = source_aliases.get(alias, {})
|
|
977
|
+
reindex_alias_data = reindex_aliases.get(alias, {})
|
|
978
|
+
|
|
979
|
+
add_alias_data = {"index": index, "alias": alias}
|
|
980
|
+
add_alias_data.update(source_alias_data)
|
|
981
|
+
if (
|
|
982
|
+
alias == self.name
|
|
983
|
+
or source_alias_data.get("is_write_index", False)
|
|
984
|
+
or reindex_alias_data.get("is_write_index", False)
|
|
985
|
+
):
|
|
986
|
+
add_alias_data["is_write_index"] = True
|
|
987
|
+
|
|
988
|
+
alias_actions.append({"add": add_alias_data})
|
|
989
|
+
|
|
990
|
+
for alias in reindex_aliases:
|
|
991
|
+
alias_actions.append({"remove": {"index": new_name, "alias": alias}})
|
|
992
|
+
|
|
993
|
+
if alias_actions:
|
|
994
|
+
logger.warning("Restoring aliases to %s", index)
|
|
995
|
+
self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
|
|
996
|
+
|
|
997
|
+
logger.warning("Deleting leftover reindex index %s", new_name)
|
|
998
|
+
self.with_retries(self.datastore.client.indices.delete, index=new_name)
|
|
999
|
+
|
|
1000
|
+
logger.warning("Unblock write to the index")
|
|
1001
|
+
self.with_retries(
|
|
1002
|
+
self.datastore.client.indices.put_settings,
|
|
1003
|
+
index=index,
|
|
1004
|
+
settings=write_unblock_settings,
|
|
1005
|
+
)
|
|
823
1006
|
|
|
824
1007
|
return True
|
|
825
1008
|
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
DELAY = 5
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
parser = argparse.ArgumentParser(
|
|
12
|
+
description="Reindex elasticsearch indexes.",
|
|
13
|
+
epilog="Valid index names are derived from the datastore configuration.",
|
|
14
|
+
)
|
|
15
|
+
parser.add_argument("indexes", nargs="*", help="Indexes to reindex.")
|
|
16
|
+
parser.add_argument("--all", action="store_true", help="Reindex all indexes.")
|
|
17
|
+
parser.add_argument("--force", action="store_true", help="Skip confirmation prompts and countdown.")
|
|
18
|
+
parser.add_argument("--verbose", action="store_true", help="Print index schema before reindexing.")
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--cleanup",
|
|
21
|
+
action="store_true",
|
|
22
|
+
help="Recover from a failed or interrupted reindex by restoring the source index and "
|
|
23
|
+
"deleting leftover '__reindex' indexes. If the source index is missing, cleanup fails "
|
|
24
|
+
"to avoid deleting the only remaining copy of the data.",
|
|
25
|
+
)
|
|
26
|
+
parser.add_argument(
|
|
27
|
+
"--allow-failures",
|
|
28
|
+
action="store_true",
|
|
29
|
+
help="DESTRUCTIVE: proceed even if the reindex reports document failures, version conflicts, "
|
|
30
|
+
"or a document count mismatch. Documents that cannot be converted to the new mappings will be "
|
|
31
|
+
"permanently dropped. Only use this for intentional lossy migrations.",
|
|
32
|
+
)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--timeout",
|
|
35
|
+
type=int,
|
|
36
|
+
default=3600,
|
|
37
|
+
help="Elasticsearch transport timeout in seconds for synchronous operations. Increase this if "
|
|
38
|
+
"large indexes time out during the clone/settings steps. Default: 3600.",
|
|
39
|
+
)
|
|
40
|
+
args = parser.parse_args()
|
|
41
|
+
|
|
42
|
+
if args.all and args.indexes:
|
|
43
|
+
parser.error("--all cannot be combined with positional index arguments.")
|
|
44
|
+
|
|
45
|
+
if not args.indexes and not args.all:
|
|
46
|
+
parser.error("Provide index names as arguments, or use --all.")
|
|
47
|
+
|
|
48
|
+
if args.timeout <= 0:
|
|
49
|
+
parser.error("--timeout must be a positive number of seconds.")
|
|
50
|
+
|
|
51
|
+
# Raise the Elasticsearch transport timeout before the datastore is imported so the value is
|
|
52
|
+
# baked into every connection (including connections rebuilt after a reset). This lets long
|
|
53
|
+
# running synchronous operations complete without timing out the client.
|
|
54
|
+
os.environ["HWL_DATASTORE_TRANSPORT_TIMEOUT"] = str(args.timeout)
|
|
55
|
+
|
|
56
|
+
from howler.datastore.collection import ESCollection
|
|
57
|
+
from howler.datastore.exceptions import DataStoreException
|
|
58
|
+
|
|
59
|
+
ESCollection.IGNORE_ENSURE_COLLECTION = True
|
|
60
|
+
|
|
61
|
+
if args.force:
|
|
62
|
+
ESCollection.ENSURE_COLLECTION_WARNED = True
|
|
63
|
+
|
|
64
|
+
from howler.common import loader
|
|
65
|
+
|
|
66
|
+
ds = loader.datastore(archive_access=False)
|
|
67
|
+
|
|
68
|
+
# Derive the set of reindexable indexes from the datastore configuration. Collections without
|
|
69
|
+
# an ODM model (e.g. user_avatar) cannot be reindexed and are excluded.
|
|
70
|
+
index_names = sorted(name for name, model in ds.ds.get_models().items() if model is not None)
|
|
71
|
+
|
|
72
|
+
invalid = [name for name in args.indexes if name not in index_names]
|
|
73
|
+
if invalid:
|
|
74
|
+
parser.error(f"Invalid index(es): {', '.join(invalid)}. Valid options: {', '.join(index_names)}")
|
|
75
|
+
|
|
76
|
+
selected = list(dict.fromkeys(index_names if args.all else args.indexes))
|
|
77
|
+
|
|
78
|
+
if args.cleanup:
|
|
79
|
+
for index_name in selected:
|
|
80
|
+
collection: ESCollection = getattr(ds, index_name)
|
|
81
|
+
print(f"Cleaning up leftover reindex state for '{index_name}'.")
|
|
82
|
+
try:
|
|
83
|
+
collection.reindex_cleanup()
|
|
84
|
+
except DataStoreException as e:
|
|
85
|
+
print(f"ERROR: Cleanup of '{index_name}' failed: {e}", file=sys.stderr)
|
|
86
|
+
sys.exit(1)
|
|
87
|
+
print(f"Cleanup of '{index_name}' complete.")
|
|
88
|
+
sys.exit(0)
|
|
89
|
+
|
|
90
|
+
if args.allow_failures and not args.force:
|
|
91
|
+
print(
|
|
92
|
+
"WARNING: --allow-failures is DESTRUCTIVE. Documents that cannot be converted to the new "
|
|
93
|
+
"mappings will be permanently dropped, and count mismatches will be ignored."
|
|
94
|
+
)
|
|
95
|
+
answer = input("Are you sure you want to proceed with --allow-failures? [yes/NO] ")
|
|
96
|
+
if not answer.startswith("y"):
|
|
97
|
+
print("Confirmation not provided, aborting.")
|
|
98
|
+
sys.exit(1)
|
|
99
|
+
|
|
100
|
+
for index_name in selected:
|
|
101
|
+
collection: ESCollection = getattr(ds, index_name)
|
|
102
|
+
|
|
103
|
+
if args.verbose:
|
|
104
|
+
print(f"Index schema for '{index_name}':")
|
|
105
|
+
print(json.dumps(collection._get_index_mappings(), indent=2))
|
|
106
|
+
|
|
107
|
+
print(f"Reindexing: {', '.join(collection.index_list_full)}")
|
|
108
|
+
|
|
109
|
+
if not args.force:
|
|
110
|
+
answer = input(f"Are you sure you want to reindex '{index_name}'? [yes/NO] ")
|
|
111
|
+
if not answer.startswith("y"):
|
|
112
|
+
print("Confirmation not provided, skipping.")
|
|
113
|
+
continue
|
|
114
|
+
|
|
115
|
+
for i in range(2 * DELAY):
|
|
116
|
+
print(f"Reindexing in {2 * DELAY - i}...", end="\r")
|
|
117
|
+
time.sleep(1)
|
|
118
|
+
print()
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
result = collection.reindex(allow_failures=args.allow_failures, request_timeout=args.timeout)
|
|
122
|
+
except Exception as e: # noqa: BLE001
|
|
123
|
+
print(f"ERROR: Reindex of '{index_name}' failed: {e}", file=sys.stderr)
|
|
124
|
+
print(
|
|
125
|
+
"Run this script with --cleanup to recover before retrying: if the source index is "
|
|
126
|
+
"still present it will be restored and any leftover '__reindex' index removed. If the "
|
|
127
|
+
"source was already deleted but '__reindex' exists, cleanup will fail to avoid deleting "
|
|
128
|
+
"the only remaining copy of the data; recover or delete that index manually. Investigate "
|
|
129
|
+
"the failure before retrying.",
|
|
130
|
+
file=sys.stderr,
|
|
131
|
+
)
|
|
132
|
+
sys.exit(1)
|
|
133
|
+
|
|
134
|
+
print(f"Reindex of '{index_name}' complete. Success: {result}.")
|
|
@@ -24,13 +24,16 @@ def assess_hit(
|
|
|
24
24
|
assessment: Optional[str] = None,
|
|
25
25
|
rationale: Optional[str] = None,
|
|
26
26
|
hit: Optional[Union[dict[str, Any], Hit]] = None,
|
|
27
|
+
*,
|
|
28
|
+
user: User | str,
|
|
27
29
|
**kwargs,
|
|
28
30
|
) -> list[OdmUpdateOperation]:
|
|
29
31
|
"""Update the assessment and esclation of a hit
|
|
30
32
|
|
|
31
33
|
Args:
|
|
34
|
+
user (User | str): The user making the assessment
|
|
32
35
|
assessment (Optional[str], optional): The assessment to set the hit to. Defaults to None.
|
|
33
|
-
hit (Optional[dict[str, Any]], optional): The hit to update. Defaults to None.
|
|
36
|
+
hit (Optional[Union[dict[str, Any], Hit]], optional): The hit to update. Defaults to None.
|
|
34
37
|
|
|
35
38
|
Raises:
|
|
36
39
|
InvalidDataException: An invalid assessment was provided
|
|
@@ -63,10 +66,16 @@ def assess_hit(
|
|
|
63
66
|
escalation,
|
|
64
67
|
)
|
|
65
68
|
|
|
69
|
+
if assessment is None:
|
|
70
|
+
assessor_id = None
|
|
71
|
+
else:
|
|
72
|
+
assessor_id = user.get("uname", user.get("username", None)) if isinstance(user, User) else user
|
|
73
|
+
|
|
66
74
|
return [
|
|
67
75
|
odm_helper.update("howler.assessment", assessment),
|
|
68
76
|
odm_helper.update("howler.escalation", escalation),
|
|
69
77
|
odm_helper.update("howler.rationale", rationale, silent=True),
|
|
78
|
+
odm_helper.update("howler.assessor", assessor_id, silent=True),
|
|
70
79
|
]
|
|
71
80
|
|
|
72
81
|
|
|
@@ -168,6 +168,7 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[str], pr
|
|
|
168
168
|
hit.howler.status = "open"
|
|
169
169
|
hit.howler.assignment = "unassigned"
|
|
170
170
|
hit.howler.escalation = choice([Escalation.HIT, Escalation.ALERT])
|
|
171
|
+
hit.howler.assessor = None
|
|
171
172
|
|
|
172
173
|
if randint(1, 10) > 9:
|
|
173
174
|
hit.howler.expiry = datetime.now() + timedelta(days=randint(1, 60))
|
|
@@ -210,6 +210,12 @@ class HowlerData(odm.Model):
|
|
|
210
210
|
description="Unique identifier of the assigned user.",
|
|
211
211
|
default=DEFAULT_ASSIGNMENT,
|
|
212
212
|
)
|
|
213
|
+
assessor: Optional[str] = odm.Optional(
|
|
214
|
+
odm.Keyword(
|
|
215
|
+
description="The most recent person to assess a hit",
|
|
216
|
+
default=None,
|
|
217
|
+
)
|
|
218
|
+
)
|
|
213
219
|
bundles: list[str] = odm.List(
|
|
214
220
|
odm.Keyword(
|
|
215
221
|
description="A list of bundle IDs this hit is a part of. Corresponds to the howler.id of the bundle."
|
|
@@ -111,7 +111,7 @@ def get_hit_workflow() -> Workflow:
|
|
|
111
111
|
"source": [HitStatus.OPEN, HitStatus.IN_PROGRESS],
|
|
112
112
|
"transition": HitStatusTransition.ASSESS,
|
|
113
113
|
"dest": HitStatus.RESOLVED,
|
|
114
|
-
"actions": [assess_hit
|
|
114
|
+
"actions": [assess_hit],
|
|
115
115
|
}
|
|
116
116
|
),
|
|
117
117
|
Transition(
|
|
@@ -152,7 +152,7 @@ suppress-none-returning = true
|
|
|
152
152
|
[tool.poetry]
|
|
153
153
|
package-mode = true
|
|
154
154
|
name = "howler-api"
|
|
155
|
-
version = "4.0.0.
|
|
155
|
+
version = "4.0.0.dev982"
|
|
156
156
|
description = "Howler - API server"
|
|
157
157
|
authors = [
|
|
158
158
|
"Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
|