howler-api 4.0.0.dev679__tar.gz → 4.0.0.dev740__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/PKG-INFO +2 -2
- howler_api-4.0.0.dev740/howler/actions/add_to_case.py +136 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/auth.py +4 -4
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/clue.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/hit.py +2 -2
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/tool.py +3 -3
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/app.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/loader.py +2 -2
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/cronjobs/retention.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/cronjobs/view_cleanup.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/collection.py +32 -23
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/migrations/fix_process.py +3 -3
- howler_api-4.0.0.dev740/howler/external/README.md +30 -0
- howler_api-4.0.0.dev740/howler/external/reindex_data.py +64 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/discover.py +3 -3
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/helper.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/random_data.py +7 -7
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/__init__.py +1 -1
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/security/__init__.py +4 -4
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/security/socket.py +4 -4
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/event_service.py +3 -3
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/pyproject.toml +3 -2
- howler_api-4.0.0.dev679/howler/external/reindex_data.py +0 -66
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/README.md +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/add_label.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/change_field.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/demote.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/example_plugin.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/prioritization.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/promote.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/remove_label.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/actions/transition.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/base.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/socket.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/action.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/analytic.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/configs.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/dossier.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/help.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/notebook.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/overview.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/search.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/template.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/user.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/utils/etag.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v1/view.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v2/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v2/case.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v2/ingest.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/api/v2/search.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/README.md +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/classification.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/classification.yml +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/exceptions.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/logging/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/logging/audit.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/logging/format.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/net.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/net_static.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/random_user.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/common/swagger.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/config.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/cronjobs/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/README.md +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/bulk.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/constants.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/exceptions.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/howler_store.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/operations.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/schemas.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/store.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/support/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/support/build.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/support/schemas.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/types.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/error.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/external/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/external/generate_mitre.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/external/generate_sigma_rules.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/external/generate_tlds.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/external/wipe_databases.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/gunicorn_config.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/healthz.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/azure.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/hit.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/oauth.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/search.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/workflow.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/helper/ws.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/README.md +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/base.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/charter.txt +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/constants.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/howler_enum.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/mixins.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/action.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/analytic.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/assemblyline.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/aws.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/azure.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/case.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/cbs.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/clue.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/config.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/dossier.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/agent.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/autonomous_system.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/client.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/cloud.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/code_signature.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/container.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/dns.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/egress.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/elf.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/email.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/error.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/event.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/faas.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/file.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/geo.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/group.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/hash.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/host.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/http.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/ingress.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/interface.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/network.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/observer.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/organization.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/os.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/pe.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/process.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/registry.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/related.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/rule.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/server.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/threat.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/tls.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/url.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/user.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/user_agent.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/ecs/vulnerability.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/gcp.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/hit.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/howler_data.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/lead.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/localized_label.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/observable.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/overview.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/pivot.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/record.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/template.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/user.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/models/view.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/odm/randomizer.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/patched.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/plugins/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/plugins/config.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/README.md +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/counters.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/events.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/hash.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/lock.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/queues/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/queues/comms.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/queues/multi.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/queues/named.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/queues/priority.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/set.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/remote/datatypes/user_quota_tracker.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/security/utils.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/action_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/analytic_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/auth_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/case_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/config_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/docs_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/dossier_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/hit_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/jwt_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/lucene_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/notebook_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/observable_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/overview_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/search_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/template_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/services/user_service.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/annotations.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/chunk.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/compat.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/dict_utils.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/isotime.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/list_utils.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/lucene.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/path.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/socket_utils.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/str_utils.py +0 -0
- {howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/utils/uid.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: howler-api
|
|
3
|
-
Version: 4.0.0.
|
|
3
|
+
Version: 4.0.0.dev740
|
|
4
4
|
Summary: Howler - API server
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
|
|
@@ -49,7 +49,7 @@ Requires-Dist: pytz (>=2025.2,<2026.0)
|
|
|
49
49
|
Requires-Dist: pyyaml (==6.0.3)
|
|
50
50
|
Requires-Dist: redis (==4.6.0)
|
|
51
51
|
Requires-Dist: requests (==2.32.5)
|
|
52
|
-
Requires-Dist:
|
|
52
|
+
Requires-Dist: tzdata (>=2026.1,<2027.0)
|
|
53
53
|
Requires-Dist: validators (>=0.34,<0.36)
|
|
54
54
|
Requires-Dist: wsproto (==1.2.0)
|
|
55
55
|
Project-URL: Documentation, https://cybercentrecanada.github.io/howler/developer/backend/
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
import chevron
|
|
4
|
+
|
|
5
|
+
from howler.common.exceptions import InvalidDataException, NotFoundException
|
|
6
|
+
from howler.common.loader import datastore
|
|
7
|
+
from howler.odm.models.action import VALID_TRIGGERS
|
|
8
|
+
from howler.services import case_service
|
|
9
|
+
|
|
10
|
+
OPERATION_ID = "add_to_case"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def execute(
|
|
14
|
+
query: str,
|
|
15
|
+
case_id: Optional[str] = None,
|
|
16
|
+
path: str = "related",
|
|
17
|
+
title_template: str = "{{howler.analytic}} ({{howler.id}})",
|
|
18
|
+
**kwargs,
|
|
19
|
+
):
|
|
20
|
+
"""Add matching alerts to a given case.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
query (str): The query on which to apply this automation.
|
|
24
|
+
case_id (str): The ID of the case to add the alerts to.
|
|
25
|
+
path (str): The path within the case at which to place the alerts. Defaults to "related".
|
|
26
|
+
title_template (str): A Mustache-compatible template string used to generate each item's
|
|
27
|
+
path suffix (title). The hit's fields are available as template variables.
|
|
28
|
+
Defaults to "{{howler.analytic}} ({{howler.id}})".
|
|
29
|
+
"""
|
|
30
|
+
if not case_id:
|
|
31
|
+
return [
|
|
32
|
+
{
|
|
33
|
+
"query": query,
|
|
34
|
+
"outcome": "error",
|
|
35
|
+
"title": "Missing Case ID",
|
|
36
|
+
"message": "A case_id must be provided.",
|
|
37
|
+
}
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
ds = datastore()
|
|
41
|
+
|
|
42
|
+
if ds.case.get(case_id) is None:
|
|
43
|
+
return [
|
|
44
|
+
{
|
|
45
|
+
"query": query,
|
|
46
|
+
"outcome": "error",
|
|
47
|
+
"title": "Case Not Found",
|
|
48
|
+
"message": f"No case with ID '{case_id}' exists.",
|
|
49
|
+
}
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
hits = ds.hit.search(query, rows=1000)["items"]
|
|
53
|
+
|
|
54
|
+
if not hits:
|
|
55
|
+
return [
|
|
56
|
+
{
|
|
57
|
+
"query": query,
|
|
58
|
+
"outcome": "skipped",
|
|
59
|
+
"title": "No Matching Hits",
|
|
60
|
+
"message": "No hits matched the query, so the action was skipped.",
|
|
61
|
+
}
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
report = []
|
|
65
|
+
skipped = []
|
|
66
|
+
added = []
|
|
67
|
+
|
|
68
|
+
normalized_path = path.rstrip("/")
|
|
69
|
+
|
|
70
|
+
for hit in hits:
|
|
71
|
+
hit_data = hit.as_primitives()
|
|
72
|
+
title = chevron.render(title_template, hit_data)
|
|
73
|
+
item_path = f"{normalized_path}/{title}" if normalized_path else title
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
case_service.append_case_item(
|
|
77
|
+
case_id,
|
|
78
|
+
item_type="hit",
|
|
79
|
+
item_value=hit.howler.id,
|
|
80
|
+
item_path=item_path,
|
|
81
|
+
)
|
|
82
|
+
added.append(hit.howler.id)
|
|
83
|
+
except InvalidDataException as e:
|
|
84
|
+
skipped.append(f"{hit.howler.id}: {e}")
|
|
85
|
+
except NotFoundException as e:
|
|
86
|
+
skipped.append(f"{hit.howler.id}: {e}")
|
|
87
|
+
except Exception as e:
|
|
88
|
+
skipped.append(f"{hit.howler.id}: {e}")
|
|
89
|
+
|
|
90
|
+
if added:
|
|
91
|
+
report.append(
|
|
92
|
+
{
|
|
93
|
+
"query": f"howler.id:({' OR '.join(added)})",
|
|
94
|
+
"outcome": "success",
|
|
95
|
+
"title": "Added to Case",
|
|
96
|
+
"message": f"{len(added)} alert(s) successfully added to case '{case_id}'.",
|
|
97
|
+
}
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if skipped:
|
|
101
|
+
report.append(
|
|
102
|
+
{
|
|
103
|
+
"query": query,
|
|
104
|
+
"outcome": "skipped",
|
|
105
|
+
"title": "Skipped Alerts",
|
|
106
|
+
"message": f"{len(skipped)} alert(s) could not be added: {'; '.join(skipped)}",
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
return report
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def specification():
|
|
114
|
+
"""Specify various properties of the action, such as title, descriptions, permissions and input steps."""
|
|
115
|
+
return {
|
|
116
|
+
"id": OPERATION_ID,
|
|
117
|
+
"title": "Add to Case",
|
|
118
|
+
"priority": 9,
|
|
119
|
+
"i18nKey": f"operations.{OPERATION_ID}",
|
|
120
|
+
"description": {
|
|
121
|
+
"short": "Add matching alerts to a case",
|
|
122
|
+
"long": execute.__doc__,
|
|
123
|
+
},
|
|
124
|
+
"roles": ["automation_basic"],
|
|
125
|
+
"steps": [
|
|
126
|
+
{
|
|
127
|
+
"args": {
|
|
128
|
+
"case_id": [],
|
|
129
|
+
"path": [],
|
|
130
|
+
"title_template": [],
|
|
131
|
+
},
|
|
132
|
+
"options": {},
|
|
133
|
+
}
|
|
134
|
+
],
|
|
135
|
+
"triggers": VALID_TRIGGERS,
|
|
136
|
+
}
|
|
@@ -347,23 +347,23 @@ def login(**_): # noqa: C901
|
|
|
347
347
|
# For sanity's sake, we throw exceptions throughout the authentication code and simply catch the exceptions here to
|
|
348
348
|
# return the corresponding HTTP Code to the user
|
|
349
349
|
except (OAuthError, AuthenticationException) as err:
|
|
350
|
-
logger.warning(
|
|
350
|
+
logger.warning("Authentication failure. (U:%s - IP:%s) [%s]", user, ip, err)
|
|
351
351
|
return unauthorized(err=str(err))
|
|
352
352
|
|
|
353
353
|
except AccessDeniedException as err:
|
|
354
|
-
logger.warning(
|
|
354
|
+
logger.warning("Authorization failure. (U:%s - IP:%s) [%s]", user, ip, err)
|
|
355
355
|
return forbidden(err=err.message)
|
|
356
356
|
|
|
357
357
|
except InvalidDataException as err:
|
|
358
358
|
return bad_request(err=err.message or str(err))
|
|
359
359
|
|
|
360
360
|
except HowlerException:
|
|
361
|
-
logger.exception(
|
|
361
|
+
logger.exception("Internal Authentication Error. (U:%s - IP:%s)", user, ip)
|
|
362
362
|
return internal_error(
|
|
363
363
|
err="Unhandled exception occured while Authenticating. Contact your administrator.",
|
|
364
364
|
)
|
|
365
365
|
|
|
366
|
-
logger.info(
|
|
366
|
+
logger.info("Login successful. (U:%s - IP:%s)", logged_in_uname, ip)
|
|
367
367
|
|
|
368
368
|
xsrf_token = generate_random_secret()
|
|
369
369
|
|
|
@@ -91,7 +91,7 @@ def proxy_to_clue(path, **kwargs):
|
|
|
91
91
|
timeout=5 * 60,
|
|
92
92
|
)
|
|
93
93
|
|
|
94
|
-
logger.debug(
|
|
94
|
+
logger.debug("Request to clue completed in %s ms", round(time.perf_counter() - start))
|
|
95
95
|
|
|
96
96
|
if not response.ok:
|
|
97
97
|
return bad_gateway(response.json(), err="Something went wrong when connecting to clue")
|
|
@@ -99,7 +99,7 @@ def create_hits(user: User, **kwargs):
|
|
|
99
99
|
response_body: dict[str, list[Any]] = {"valid": [], "invalid": []}
|
|
100
100
|
odms = []
|
|
101
101
|
ignore_extra_values: bool = bool(request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true"))
|
|
102
|
-
logger.debug(
|
|
102
|
+
logger.debug("ignore_extra_values = %s", ignore_extra_values)
|
|
103
103
|
warnings = []
|
|
104
104
|
for hit in hits:
|
|
105
105
|
try:
|
|
@@ -108,7 +108,7 @@ def create_hits(user: User, **kwargs):
|
|
|
108
108
|
odms.append(odm)
|
|
109
109
|
warnings.extend(_warnings)
|
|
110
110
|
except HowlerException as e:
|
|
111
|
-
logger.warning(
|
|
111
|
+
logger.warning("%s when saving new hit!", type(e).__name__)
|
|
112
112
|
logger.warning(e)
|
|
113
113
|
response_body["invalid"].append({"input": hit, "error": str(e)})
|
|
114
114
|
|
|
@@ -67,7 +67,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
67
67
|
field_map = data.pop("map", None)
|
|
68
68
|
hits = data.pop("hits", None)
|
|
69
69
|
ignore_extra_values: bool = bool(request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true"))
|
|
70
|
-
logger.debug(
|
|
70
|
+
logger.debug("ignore_extra_values = %s", ignore_extra_values)
|
|
71
71
|
# Check data type
|
|
72
72
|
if not isinstance(field_map, dict):
|
|
73
73
|
return bad_request(err="Invalid: 'map' field is missing or invalid.")
|
|
@@ -114,7 +114,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
114
114
|
try:
|
|
115
115
|
field_data: Optional[_Field] = hit_fields[target]
|
|
116
116
|
except KeyError:
|
|
117
|
-
logger.debug(
|
|
117
|
+
logger.debug("`%s` not in hit fields", target)
|
|
118
118
|
field_data = next(
|
|
119
119
|
(v for k, v in hit_fields.items() if get_parent_key(k) == target),
|
|
120
120
|
None,
|
|
@@ -147,7 +147,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
147
147
|
}
|
|
148
148
|
)
|
|
149
149
|
except HowlerException as e:
|
|
150
|
-
logger.warning(
|
|
150
|
+
logger.warning("%s when saving %s!", type(e).__name__, cur_id)
|
|
151
151
|
logger.warning(e)
|
|
152
152
|
|
|
153
153
|
out.append({"id": None, "error": str(e)})
|
|
@@ -216,7 +216,7 @@ if logger.parent:
|
|
|
216
216
|
|
|
217
217
|
# Setup APMs
|
|
218
218
|
if config.core.metrics.apm_server.server_url is not None:
|
|
219
|
-
logger.info(
|
|
219
|
+
logger.info("Exporting application metrics to: %s", config.core.metrics.apm_server.server_url)
|
|
220
220
|
ElasticAPM(
|
|
221
221
|
app,
|
|
222
222
|
server_url=config.core.metrics.apm_server.server_url,
|
|
@@ -60,9 +60,9 @@ def get_classification(yml_config: Optional[str] = None): # noqa: C901
|
|
|
60
60
|
)
|
|
61
61
|
|
|
62
62
|
if not yml_config_path.exists():
|
|
63
|
-
log.warning(
|
|
63
|
+
log.warning("%s does not exist!", yml_config_path)
|
|
64
64
|
yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "classification.yml"
|
|
65
|
-
log.warning(
|
|
65
|
+
log.warning("Checking at %s instead.", yml_config_path)
|
|
66
66
|
else:
|
|
67
67
|
yml_config_path = Path(yml_config)
|
|
68
68
|
|
|
@@ -39,7 +39,7 @@ def setup_job(sched: BaseScheduler):
|
|
|
39
39
|
|
|
40
40
|
return
|
|
41
41
|
|
|
42
|
-
logger.debug(
|
|
42
|
+
logger.debug("Initializing retention cronjob with cron %s", config.system.retention.crontab)
|
|
43
43
|
|
|
44
44
|
if DEBUG:
|
|
45
45
|
_kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
|
|
@@ -68,7 +68,7 @@ def setup_job(sched: BaseScheduler):
|
|
|
68
68
|
|
|
69
69
|
return
|
|
70
70
|
|
|
71
|
-
logger.debug(
|
|
71
|
+
logger.debug("Initializing view cleanup cronjob with cron %s", config.system.view_cleanup.crontab)
|
|
72
72
|
|
|
73
73
|
if DEBUG:
|
|
74
74
|
_kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
|
|
@@ -398,7 +398,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
398
398
|
except elasticsearch.exceptions.TransportError as e:
|
|
399
399
|
err_code, msg, cause = e.args
|
|
400
400
|
if err_code == 503 or err_code == "503":
|
|
401
|
-
logger.warning(
|
|
401
|
+
logger.warning("Looks like index %s is not ready yet, retrying...", self.name)
|
|
402
402
|
time.sleep(min(retries, self.MAX_RETRY_BACKOFF))
|
|
403
403
|
self.datastore.connection_reset()
|
|
404
404
|
retries += 1
|
|
@@ -411,7 +411,8 @@ class ESCollection(Generic[ModelType]):
|
|
|
411
411
|
retries += 1
|
|
412
412
|
elif err_code == 403 or err_code == "403":
|
|
413
413
|
logger.warning(
|
|
414
|
-
|
|
414
|
+
"Elasticsearch cluster is preventing writing operations on index %s, retrying...",
|
|
415
|
+
self.name,
|
|
415
416
|
)
|
|
416
417
|
time.sleep(min(retries, self.MAX_RETRY_BACKOFF))
|
|
417
418
|
self.datastore.connection_reset()
|
|
@@ -470,7 +471,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
470
471
|
except elasticsearch.exceptions.TransportError as e:
|
|
471
472
|
err_code, _, _ = e.args
|
|
472
473
|
if err_code == 408 or err_code == "408":
|
|
473
|
-
logger.warning(
|
|
474
|
+
logger.warning("Waiting for index %s to get to status %s...", index, min_status)
|
|
474
475
|
else:
|
|
475
476
|
raise
|
|
476
477
|
|
|
@@ -576,8 +577,9 @@ class ESCollection(Generic[ModelType]):
|
|
|
576
577
|
|
|
577
578
|
if cur_shards > target_shards:
|
|
578
579
|
logger.info(
|
|
579
|
-
|
|
580
|
-
|
|
580
|
+
"Current shards (%s) is bigger then target shards (%s), we will be shrinking the index.",
|
|
581
|
+
cur_shards,
|
|
582
|
+
target_shards,
|
|
581
583
|
)
|
|
582
584
|
if cur_shards % target_shards != 0:
|
|
583
585
|
logger.info("The target shards is not a factor of the current shards, aborting...")
|
|
@@ -591,8 +593,9 @@ class ESCollection(Generic[ModelType]):
|
|
|
591
593
|
method = self.datastore.client.indices.shrink
|
|
592
594
|
elif cur_shards < target_shards:
|
|
593
595
|
logger.info(
|
|
594
|
-
|
|
595
|
-
|
|
596
|
+
"Current shards (%s) is smaller then target shards (%s), we will be splitting the index.",
|
|
597
|
+
cur_shards,
|
|
598
|
+
target_shards,
|
|
596
599
|
)
|
|
597
600
|
if target_shards % cur_shards != 0:
|
|
598
601
|
logger.warning("The current shards is not a factor of the target shards, aborting...")
|
|
@@ -601,13 +604,15 @@ class ESCollection(Generic[ModelType]):
|
|
|
601
604
|
method = self.datastore.client.indices.split
|
|
602
605
|
else:
|
|
603
606
|
logger.info(
|
|
604
|
-
|
|
605
|
-
"
|
|
607
|
+
"Current shards (%s) is equal to the target shards (%s), only housekeeping operations will be "
|
|
608
|
+
"performed.",
|
|
609
|
+
cur_shards,
|
|
610
|
+
target_shards,
|
|
606
611
|
)
|
|
607
612
|
|
|
608
613
|
if method:
|
|
609
614
|
# Before we do anything, we should make sure the source index is in a good state
|
|
610
|
-
logger.info(
|
|
615
|
+
logger.info("Waiting for %s status to be GREEN.", self.name.upper())
|
|
611
616
|
self._wait_for_status(self.name, min_status="green")
|
|
612
617
|
|
|
613
618
|
# Block all indexes to be written to
|
|
@@ -618,7 +623,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
618
623
|
if not self.with_retries(self.datastore.client.indices.exists, index=temp_name):
|
|
619
624
|
# if there are specific settings to be applied to the index, apply them
|
|
620
625
|
if clone_setup_settings:
|
|
621
|
-
logger.info(
|
|
626
|
+
logger.info("Relocating index to node %s.", target_node.upper())
|
|
622
627
|
self.with_retries(
|
|
623
628
|
self.datastore.client.indices.put_settings,
|
|
624
629
|
index=self.index_name,
|
|
@@ -630,7 +635,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
630
635
|
time.sleep(1)
|
|
631
636
|
|
|
632
637
|
# Make a clone of the current index
|
|
633
|
-
logger.info(
|
|
638
|
+
logger.info("Cloning %s into %s.", self.index_name.upper(), temp_name.upper())
|
|
634
639
|
self._safe_index_copy(
|
|
635
640
|
self.datastore.client.indices.clone,
|
|
636
641
|
self.index_name,
|
|
@@ -640,14 +645,16 @@ class ESCollection(Generic[ModelType]):
|
|
|
640
645
|
)
|
|
641
646
|
|
|
642
647
|
# Make 100% sure temporary index is ready
|
|
643
|
-
logger.info(
|
|
648
|
+
logger.info("Waiting for %s status to be GREEN.", temp_name.upper())
|
|
644
649
|
self._wait_for_status(temp_name, "green")
|
|
645
650
|
|
|
646
651
|
# Make sure temporary index is the alias if not already
|
|
647
652
|
if self._get_current_alias(self.name) != temp_name:
|
|
648
653
|
logger.info(
|
|
649
|
-
|
|
650
|
-
|
|
654
|
+
"Make %s the current alias for %s and delete %s.",
|
|
655
|
+
temp_name.upper(),
|
|
656
|
+
self.name.upper(),
|
|
657
|
+
self.index_name.upper(),
|
|
651
658
|
)
|
|
652
659
|
# Make the hot index the temporary index while deleting the original index
|
|
653
660
|
alias_actions = [
|
|
@@ -658,17 +665,19 @@ class ESCollection(Generic[ModelType]):
|
|
|
658
665
|
|
|
659
666
|
# Make sure the original index is deleted
|
|
660
667
|
if self.with_retries(self.datastore.client.indices.exists, index=self.index_name):
|
|
661
|
-
logger.info(
|
|
668
|
+
logger.info("Delete extra %s index.", self.index_name.upper())
|
|
662
669
|
self.with_retries(self.datastore.client.indices.delete, index=self.index_name)
|
|
663
670
|
|
|
664
671
|
# Shrink/split the temporary index into the original index
|
|
665
|
-
logger.info(
|
|
672
|
+
logger.info("Perform shard fix operation from %s to %s.", temp_name.upper(), self.index_name.upper())
|
|
666
673
|
self._safe_index_copy(method, temp_name, self.index_name, settings=settings)
|
|
667
674
|
|
|
668
675
|
# Make the original index the new alias
|
|
669
676
|
logger.info(
|
|
670
|
-
|
|
671
|
-
|
|
677
|
+
"Make %s the current alias for %s and delete %s.",
|
|
678
|
+
self.index_name.upper(),
|
|
679
|
+
self.name.upper(),
|
|
680
|
+
temp_name.upper(),
|
|
672
681
|
)
|
|
673
682
|
alias_actions = [
|
|
674
683
|
{"add": {"index": self.index_name, "alias": self.name}},
|
|
@@ -681,7 +690,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
681
690
|
self.with_retries(self.datastore.client.indices.put_settings, settings=write_unblock_settings)
|
|
682
691
|
|
|
683
692
|
# Restore normal routing and replicas
|
|
684
|
-
logger.debug(
|
|
693
|
+
logger.debug("Restore original routing table for %s.", self.name.upper())
|
|
685
694
|
self.with_retries(
|
|
686
695
|
self.datastore.client.indices.put_settings,
|
|
687
696
|
index=self.name,
|
|
@@ -849,7 +858,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
849
858
|
key_list.remove(row["_id"])
|
|
850
859
|
add_to_output(row["_source"], row["_id"])
|
|
851
860
|
except ValueError:
|
|
852
|
-
logger.exception(
|
|
861
|
+
logger.exception("MGet returned multiple documents for id: %s", row["_id"])
|
|
853
862
|
|
|
854
863
|
if key_list and error_on_missing:
|
|
855
864
|
raise MultiKeyError(key_list, out)
|
|
@@ -2358,7 +2367,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
2358
2367
|
"""
|
|
2359
2368
|
# Create HOT index
|
|
2360
2369
|
if not self.with_retries(self.datastore.client.indices.exists, index=self.name):
|
|
2361
|
-
logger.debug(
|
|
2370
|
+
logger.debug("Index %s does not exist. Creating it now...", self.name.upper())
|
|
2362
2371
|
try:
|
|
2363
2372
|
self.with_retries(
|
|
2364
2373
|
self.datastore.client.indices.create,
|
|
@@ -2369,7 +2378,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
2369
2378
|
except elasticsearch.exceptions.RequestError as e:
|
|
2370
2379
|
if "resource_already_exists_exception" not in str(e):
|
|
2371
2380
|
raise
|
|
2372
|
-
logger.warning(
|
|
2381
|
+
logger.warning("Tried to create an index template that already exists: %s", self.name.upper())
|
|
2373
2382
|
|
|
2374
2383
|
self.with_retries(
|
|
2375
2384
|
self.datastore.client.indices.put_alias,
|
{howler_api-4.0.0.dev679 → howler_api-4.0.0.dev740}/howler/datastore/migrations/fix_process.py
RENAMED
|
@@ -15,12 +15,12 @@ def migrate():
|
|
|
15
15
|
logger.info("Preconditions met, continuing.")
|
|
16
16
|
|
|
17
17
|
db_size = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
|
|
18
|
-
logger.info(
|
|
18
|
+
logger.info("Database size pre-migration: %s", db_size)
|
|
19
19
|
else:
|
|
20
20
|
logger.info("Preconditions not met, stopping")
|
|
21
21
|
return
|
|
22
22
|
|
|
23
|
-
logger.info(
|
|
23
|
+
logger.info("We will delete %s hits. Continue?", result["total"])
|
|
24
24
|
prompt_result = input("y/[n]")
|
|
25
25
|
|
|
26
26
|
if prompt_result.lower() != "y":
|
|
@@ -32,7 +32,7 @@ def migrate():
|
|
|
32
32
|
collection.commit()
|
|
33
33
|
|
|
34
34
|
db_size_after = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
|
|
35
|
-
logger.info(
|
|
35
|
+
logger.info("Database size post-migration: %s", db_size_after)
|
|
36
36
|
|
|
37
37
|
logger.info("Migration complete")
|
|
38
38
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# External scripts
|
|
2
|
+
|
|
3
|
+
## reindex_data.py
|
|
4
|
+
|
|
5
|
+
Reindex one or more Elasticsearch indexes used by Howler.
|
|
6
|
+
|
|
7
|
+
### Usage
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# Reindex specific indexes (confirms each one before proceeding)
|
|
11
|
+
python reindex_data.py hit user
|
|
12
|
+
|
|
13
|
+
# Reindex all indexes
|
|
14
|
+
python reindex_data.py --all
|
|
15
|
+
|
|
16
|
+
# Skip confirmation prompts and countdown
|
|
17
|
+
python reindex_data.py hit --force
|
|
18
|
+
|
|
19
|
+
# Print index schema before reindexing
|
|
20
|
+
python reindex_data.py hit --verbose
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### Options
|
|
24
|
+
|
|
25
|
+
| Argument | Description |
|
|
26
|
+
|-------------|----------------------------------------------|
|
|
27
|
+
| `indexes` | One or more index names to reindex. |
|
|
28
|
+
| `--all` | Reindex all indexes. |
|
|
29
|
+
| `--force` | Skip confirmation prompts and countdown. |
|
|
30
|
+
| `--verbose` | Print the index schema before reindexing. |
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
DELAY = 5
|
|
6
|
+
INDEX_NAMES = ["analytic", "hit", "view", "template", "overview", "action", "user", "dossier"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
if __name__ == "__main__":
|
|
10
|
+
parser = argparse.ArgumentParser(
|
|
11
|
+
description="Reindex elasticsearch indexes.",
|
|
12
|
+
epilog=f"Valid index names: {', '.join(INDEX_NAMES)}",
|
|
13
|
+
)
|
|
14
|
+
parser.add_argument("indexes", nargs="*", help="Indexes to reindex.")
|
|
15
|
+
parser.add_argument("--all", action="store_true", help="Reindex all indexes.")
|
|
16
|
+
parser.add_argument("--force", action="store_true", help="Skip confirmation prompts and countdown.")
|
|
17
|
+
parser.add_argument("--verbose", action="store_true", help="Print index schema before reindexing.")
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
|
|
20
|
+
if args.all and args.indexes:
|
|
21
|
+
parser.error("--all cannot be combined with positional index arguments.")
|
|
22
|
+
|
|
23
|
+
if not args.indexes and not args.all:
|
|
24
|
+
parser.error("Provide index names as arguments, or use --all.")
|
|
25
|
+
|
|
26
|
+
invalid = [name for name in args.indexes if name not in INDEX_NAMES]
|
|
27
|
+
if invalid:
|
|
28
|
+
parser.error(f"Invalid index(es): {', '.join(invalid)}. Valid options: {', '.join(INDEX_NAMES)}")
|
|
29
|
+
|
|
30
|
+
from howler.datastore.collection import ESCollection
|
|
31
|
+
|
|
32
|
+
ESCollection.IGNORE_ENSURE_COLLECTION = True
|
|
33
|
+
|
|
34
|
+
if args.force:
|
|
35
|
+
ESCollection.ENSURE_COLLECTION_WARNED = True
|
|
36
|
+
|
|
37
|
+
from howler.common import loader
|
|
38
|
+
|
|
39
|
+
ds = loader.datastore(archive_access=False)
|
|
40
|
+
|
|
41
|
+
selected = list(dict.fromkeys(INDEX_NAMES if args.all else args.indexes))
|
|
42
|
+
|
|
43
|
+
for index_name in selected:
|
|
44
|
+
collection: ESCollection = getattr(ds, index_name)
|
|
45
|
+
|
|
46
|
+
if args.verbose:
|
|
47
|
+
print(f"Index schema for '{index_name}':")
|
|
48
|
+
print(json.dumps(collection._get_index_mappings(), indent=2))
|
|
49
|
+
|
|
50
|
+
print(f"Reindexing: {', '.join(collection.index_list_full)}")
|
|
51
|
+
|
|
52
|
+
if not args.force:
|
|
53
|
+
answer = input(f"Are you sure you want to reindex '{index_name}'? [yes/NO] ")
|
|
54
|
+
if not answer.startswith("y"):
|
|
55
|
+
print("Confirmation not provided, skipping.")
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
for i in range(2 * DELAY):
|
|
59
|
+
print(f"Reindexing in {2 * DELAY - i}...", end="\r")
|
|
60
|
+
time.sleep(1)
|
|
61
|
+
print()
|
|
62
|
+
|
|
63
|
+
result = collection.reindex()
|
|
64
|
+
print(f"Reindex of '{index_name}' complete. Success: {result}.")
|
|
@@ -51,11 +51,11 @@ def get_apps_list(discovery_url: Optional[str]) -> list[dict[str, str]]:
|
|
|
51
51
|
)
|
|
52
52
|
|
|
53
53
|
except Exception:
|
|
54
|
-
logger.exception(
|
|
54
|
+
logger.exception("Failed to parse get app: %s", str(app))
|
|
55
55
|
else:
|
|
56
|
-
logger.warning(
|
|
56
|
+
logger.warning("Invalid response from server for apps discovery: %s", discovery_url)
|
|
57
57
|
except Exception:
|
|
58
|
-
logger.exception(
|
|
58
|
+
logger.exception("Failed to get apps from discover URL: %s", discovery_url)
|
|
59
59
|
|
|
60
60
|
DISCO_CACHE[discovery_url] = sorted(apps, key=lambda k: k["name"])
|
|
61
61
|
return sorted(apps, key=lambda k: k["name"])
|
|
@@ -471,7 +471,7 @@ def create_users_with_username(ds: HowlerDatastore, usernames: list[str]):
|
|
|
471
471
|
ds.user.save(username, user_data)
|
|
472
472
|
|
|
473
473
|
if "pytest" not in sys.modules:
|
|
474
|
-
logger.info(
|
|
474
|
+
logger.info("%s:%s", username, username)
|
|
475
475
|
|
|
476
476
|
ds.user.commit()
|
|
477
477
|
ds.user_avatar.commit()
|
|
@@ -139,7 +139,7 @@ def create_users(ds):
|
|
|
139
139
|
ds.view.save(admin_view.view_id, admin_view)
|
|
140
140
|
|
|
141
141
|
if "pytest" not in sys.modules:
|
|
142
|
-
logger.info(
|
|
142
|
+
logger.info("\t%s:%s", user_data.uname, admin_pass)
|
|
143
143
|
|
|
144
144
|
user_hash = get_password_hash(user_pass)
|
|
145
145
|
|
|
@@ -186,7 +186,7 @@ def create_users(ds):
|
|
|
186
186
|
ds.view.save(user_view.view_id, user_view)
|
|
187
187
|
|
|
188
188
|
if "pytest" not in sys.modules:
|
|
189
|
-
logger.info(
|
|
189
|
+
logger.info("\t%s:%s", user_data.uname, user_pass)
|
|
190
190
|
|
|
191
191
|
huey_hash = get_password_hash(huey_pass)
|
|
192
192
|
|
|
@@ -233,7 +233,7 @@ def create_users(ds):
|
|
|
233
233
|
ds.view.save(huey_view.view_id, huey_view)
|
|
234
234
|
|
|
235
235
|
if "pytest" not in sys.modules:
|
|
236
|
-
logger.info(
|
|
236
|
+
logger.info("\t%s:%s", huey_data.uname, huey_pass)
|
|
237
237
|
|
|
238
238
|
shawnh_view = View(
|
|
239
239
|
{
|
|
@@ -263,7 +263,7 @@ def create_users(ds):
|
|
|
263
263
|
ds.view.save(shawnh_view.view_id, shawnh_view)
|
|
264
264
|
|
|
265
265
|
if "pytest" not in sys.modules:
|
|
266
|
-
logger.info(
|
|
266
|
+
logger.info("\t%s:%s", shawn_data.uname, shawnh_pass)
|
|
267
267
|
|
|
268
268
|
goose_view = View(
|
|
269
269
|
{
|
|
@@ -293,7 +293,7 @@ def create_users(ds):
|
|
|
293
293
|
ds.view.save(goose_view.view_id, goose_view)
|
|
294
294
|
|
|
295
295
|
if "pytest" not in sys.modules:
|
|
296
|
-
logger.info(
|
|
296
|
+
logger.info("\t%s:%s", goose_data.uname, goose_pass)
|
|
297
297
|
|
|
298
298
|
ds.user.commit()
|
|
299
299
|
ds.user_avatar.commit()
|
|
@@ -1063,7 +1063,7 @@ def create_actions(ds: HowlerDatastore, num_actions: int = 30):
|
|
|
1063
1063
|
|
|
1064
1064
|
for step in available_operations[operation_id].specification()["steps"]:
|
|
1065
1065
|
for key in step["args"].keys():
|
|
1066
|
-
potential_values = step
|
|
1066
|
+
potential_values = step.get("options", {}).get(key, None)
|
|
1067
1067
|
if potential_values:
|
|
1068
1068
|
if isinstance(potential_values, dict):
|
|
1069
1069
|
try:
|
|
@@ -1189,7 +1189,7 @@ if __name__ == "__main__":
|
|
|
1189
1189
|
|
|
1190
1190
|
for index, operations in INDEXES.items():
|
|
1191
1191
|
if index in args:
|
|
1192
|
-
logger.info(
|
|
1192
|
+
logger.info("Creating %s...", index)
|
|
1193
1193
|
|
|
1194
1194
|
# Create functions
|
|
1195
1195
|
for create_fn in operations[1]:
|