howler-api 4.0.0.dev676__tar.gz → 4.0.0.dev724__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/PKG-INFO +2 -2
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/auth.py +4 -4
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/clue.py +1 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/hit.py +2 -2
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/tool.py +3 -3
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/app.py +1 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/loader.py +2 -2
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/cronjobs/retention.py +1 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/cronjobs/view_cleanup.py +1 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/collection.py +32 -23
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/migrations/fix_process.py +3 -3
- howler_api-4.0.0.dev724/howler/external/README.md +30 -0
- howler_api-4.0.0.dev724/howler/external/reindex_data.py +64 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/discover.py +3 -3
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/base.py +22 -2
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/helper.py +1 -1
- howler_api-4.0.0.dev724/howler/odm/mixins.py +97 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/case.py +2 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/hit.py +3 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/observable.py +3 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/random_data.py +6 -6
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/__init__.py +1 -1
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/security/__init__.py +4 -4
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/security/socket.py +4 -4
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/event_service.py +3 -3
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/pyproject.toml +3 -2
- howler_api-4.0.0.dev676/howler/external/reindex_data.py +0 -66
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/README.md +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/add_label.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/change_field.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/demote.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/example_plugin.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/prioritization.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/promote.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/remove_label.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/actions/transition.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/base.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/socket.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/action.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/analytic.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/configs.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/dossier.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/help.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/notebook.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/overview.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/search.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/template.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/user.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/utils/etag.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v1/view.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v2/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v2/case.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v2/ingest.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/api/v2/search.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/README.md +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/classification.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/classification.yml +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/exceptions.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/logging/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/logging/audit.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/logging/format.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/net.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/net_static.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/random_user.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/common/swagger.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/config.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/cronjobs/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/README.md +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/bulk.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/constants.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/exceptions.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/howler_store.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/operations.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/schemas.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/store.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/support/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/support/build.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/support/schemas.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/types.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/error.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/external/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/external/generate_mitre.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/external/generate_sigma_rules.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/external/generate_tlds.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/external/wipe_databases.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/gunicorn_config.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/healthz.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/azure.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/hit.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/oauth.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/search.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/workflow.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/helper/ws.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/README.md +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/charter.txt +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/constants.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/howler_enum.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/action.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/analytic.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/assemblyline.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/aws.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/azure.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/cbs.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/clue.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/config.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/dossier.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/agent.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/autonomous_system.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/client.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/cloud.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/code_signature.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/container.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/dns.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/egress.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/elf.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/email.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/error.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/event.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/faas.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/file.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/geo.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/group.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/hash.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/host.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/http.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/ingress.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/interface.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/network.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/observer.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/organization.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/os.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/pe.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/process.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/registry.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/related.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/rule.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/server.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/threat.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/tls.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/url.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/user.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/user_agent.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/ecs/vulnerability.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/gcp.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/howler_data.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/lead.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/localized_label.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/overview.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/pivot.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/record.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/template.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/user.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/models/view.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/odm/randomizer.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/patched.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/plugins/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/plugins/config.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/README.md +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/counters.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/events.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/hash.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/lock.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/queues/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/queues/comms.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/queues/multi.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/queues/named.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/queues/priority.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/set.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/remote/datatypes/user_quota_tracker.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/security/utils.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/action_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/analytic_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/auth_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/case_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/config_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/docs_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/dossier_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/hit_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/jwt_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/lucene_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/notebook_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/observable_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/overview_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/search_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/template_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/services/user_service.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/__init__.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/annotations.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/chunk.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/compat.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/dict_utils.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/isotime.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/list_utils.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/lucene.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/path.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/socket_utils.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/str_utils.py +0 -0
- {howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/utils/uid.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: howler-api
|
|
3
|
-
Version: 4.0.0.
|
|
3
|
+
Version: 4.0.0.dev724
|
|
4
4
|
Summary: Howler - API server
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
|
|
@@ -49,7 +49,7 @@ Requires-Dist: pytz (>=2025.2,<2026.0)
|
|
|
49
49
|
Requires-Dist: pyyaml (==6.0.3)
|
|
50
50
|
Requires-Dist: redis (==4.6.0)
|
|
51
51
|
Requires-Dist: requests (==2.32.5)
|
|
52
|
-
Requires-Dist:
|
|
52
|
+
Requires-Dist: tzdata (>=2026.1,<2027.0)
|
|
53
53
|
Requires-Dist: validators (>=0.34,<0.36)
|
|
54
54
|
Requires-Dist: wsproto (==1.2.0)
|
|
55
55
|
Project-URL: Documentation, https://cybercentrecanada.github.io/howler/developer/backend/
|
|
@@ -347,23 +347,23 @@ def login(**_): # noqa: C901
|
|
|
347
347
|
# For sanity's sake, we throw exceptions throughout the authentication code and simply catch the exceptions here to
|
|
348
348
|
# return the corresponding HTTP Code to the user
|
|
349
349
|
except (OAuthError, AuthenticationException) as err:
|
|
350
|
-
logger.warning(
|
|
350
|
+
logger.warning("Authentication failure. (U:%s - IP:%s) [%s]", user, ip, err)
|
|
351
351
|
return unauthorized(err=str(err))
|
|
352
352
|
|
|
353
353
|
except AccessDeniedException as err:
|
|
354
|
-
logger.warning(
|
|
354
|
+
logger.warning("Authorization failure. (U:%s - IP:%s) [%s]", user, ip, err)
|
|
355
355
|
return forbidden(err=err.message)
|
|
356
356
|
|
|
357
357
|
except InvalidDataException as err:
|
|
358
358
|
return bad_request(err=err.message or str(err))
|
|
359
359
|
|
|
360
360
|
except HowlerException:
|
|
361
|
-
logger.exception(
|
|
361
|
+
logger.exception("Internal Authentication Error. (U:%s - IP:%s)", user, ip)
|
|
362
362
|
return internal_error(
|
|
363
363
|
err="Unhandled exception occured while Authenticating. Contact your administrator.",
|
|
364
364
|
)
|
|
365
365
|
|
|
366
|
-
logger.info(
|
|
366
|
+
logger.info("Login successful. (U:%s - IP:%s)", logged_in_uname, ip)
|
|
367
367
|
|
|
368
368
|
xsrf_token = generate_random_secret()
|
|
369
369
|
|
|
@@ -91,7 +91,7 @@ def proxy_to_clue(path, **kwargs):
|
|
|
91
91
|
timeout=5 * 60,
|
|
92
92
|
)
|
|
93
93
|
|
|
94
|
-
logger.debug(
|
|
94
|
+
logger.debug("Request to clue completed in %s ms", round(time.perf_counter() - start))
|
|
95
95
|
|
|
96
96
|
if not response.ok:
|
|
97
97
|
return bad_gateway(response.json(), err="Something went wrong when connecting to clue")
|
|
@@ -99,7 +99,7 @@ def create_hits(user: User, **kwargs):
|
|
|
99
99
|
response_body: dict[str, list[Any]] = {"valid": [], "invalid": []}
|
|
100
100
|
odms = []
|
|
101
101
|
ignore_extra_values: bool = bool(request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true"))
|
|
102
|
-
logger.debug(
|
|
102
|
+
logger.debug("ignore_extra_values = %s", ignore_extra_values)
|
|
103
103
|
warnings = []
|
|
104
104
|
for hit in hits:
|
|
105
105
|
try:
|
|
@@ -108,7 +108,7 @@ def create_hits(user: User, **kwargs):
|
|
|
108
108
|
odms.append(odm)
|
|
109
109
|
warnings.extend(_warnings)
|
|
110
110
|
except HowlerException as e:
|
|
111
|
-
logger.warning(
|
|
111
|
+
logger.warning("%s when saving new hit!", type(e).__name__)
|
|
112
112
|
logger.warning(e)
|
|
113
113
|
response_body["invalid"].append({"input": hit, "error": str(e)})
|
|
114
114
|
|
|
@@ -67,7 +67,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
67
67
|
field_map = data.pop("map", None)
|
|
68
68
|
hits = data.pop("hits", None)
|
|
69
69
|
ignore_extra_values: bool = bool(request.args.get("ignore_extra_values", False, type=lambda v: v.lower() == "true"))
|
|
70
|
-
logger.debug(
|
|
70
|
+
logger.debug("ignore_extra_values = %s", ignore_extra_values)
|
|
71
71
|
# Check data type
|
|
72
72
|
if not isinstance(field_map, dict):
|
|
73
73
|
return bad_request(err="Invalid: 'map' field is missing or invalid.")
|
|
@@ -114,7 +114,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
114
114
|
try:
|
|
115
115
|
field_data: Optional[_Field] = hit_fields[target]
|
|
116
116
|
except KeyError:
|
|
117
|
-
logger.debug(
|
|
117
|
+
logger.debug("`%s` not in hit fields", target)
|
|
118
118
|
field_data = next(
|
|
119
119
|
(v for k, v in hit_fields.items() if get_parent_key(k) == target),
|
|
120
120
|
None,
|
|
@@ -147,7 +147,7 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs): # noqa: C901
|
|
|
147
147
|
}
|
|
148
148
|
)
|
|
149
149
|
except HowlerException as e:
|
|
150
|
-
logger.warning(
|
|
150
|
+
logger.warning("%s when saving %s!", type(e).__name__, cur_id)
|
|
151
151
|
logger.warning(e)
|
|
152
152
|
|
|
153
153
|
out.append({"id": None, "error": str(e)})
|
|
@@ -216,7 +216,7 @@ if logger.parent:
|
|
|
216
216
|
|
|
217
217
|
# Setup APMs
|
|
218
218
|
if config.core.metrics.apm_server.server_url is not None:
|
|
219
|
-
logger.info(
|
|
219
|
+
logger.info("Exporting application metrics to: %s", config.core.metrics.apm_server.server_url)
|
|
220
220
|
ElasticAPM(
|
|
221
221
|
app,
|
|
222
222
|
server_url=config.core.metrics.apm_server.server_url,
|
|
@@ -60,9 +60,9 @@ def get_classification(yml_config: Optional[str] = None): # noqa: C901
|
|
|
60
60
|
)
|
|
61
61
|
|
|
62
62
|
if not yml_config_path.exists():
|
|
63
|
-
log.warning(
|
|
63
|
+
log.warning("%s does not exist!", yml_config_path)
|
|
64
64
|
yml_config_path = Path("/etc") / APP_NAME.replace("-dev", "") / "classification.yml"
|
|
65
|
-
log.warning(
|
|
65
|
+
log.warning("Checking at %s instead.", yml_config_path)
|
|
66
66
|
else:
|
|
67
67
|
yml_config_path = Path(yml_config)
|
|
68
68
|
|
|
@@ -39,7 +39,7 @@ def setup_job(sched: BaseScheduler):
|
|
|
39
39
|
|
|
40
40
|
return
|
|
41
41
|
|
|
42
|
-
logger.debug(
|
|
42
|
+
logger.debug("Initializing retention cronjob with cron %s", config.system.retention.crontab)
|
|
43
43
|
|
|
44
44
|
if DEBUG:
|
|
45
45
|
_kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
|
|
@@ -68,7 +68,7 @@ def setup_job(sched: BaseScheduler):
|
|
|
68
68
|
|
|
69
69
|
return
|
|
70
70
|
|
|
71
|
-
logger.debug(
|
|
71
|
+
logger.debug("Initializing view cleanup cronjob with cron %s", config.system.view_cleanup.crontab)
|
|
72
72
|
|
|
73
73
|
if DEBUG:
|
|
74
74
|
_kwargs: dict[str, Any] = {"next_run_time": datetime.now()}
|
|
@@ -398,7 +398,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
398
398
|
except elasticsearch.exceptions.TransportError as e:
|
|
399
399
|
err_code, msg, cause = e.args
|
|
400
400
|
if err_code == 503 or err_code == "503":
|
|
401
|
-
logger.warning(
|
|
401
|
+
logger.warning("Looks like index %s is not ready yet, retrying...", self.name)
|
|
402
402
|
time.sleep(min(retries, self.MAX_RETRY_BACKOFF))
|
|
403
403
|
self.datastore.connection_reset()
|
|
404
404
|
retries += 1
|
|
@@ -411,7 +411,8 @@ class ESCollection(Generic[ModelType]):
|
|
|
411
411
|
retries += 1
|
|
412
412
|
elif err_code == 403 or err_code == "403":
|
|
413
413
|
logger.warning(
|
|
414
|
-
|
|
414
|
+
"Elasticsearch cluster is preventing writing operations on index %s, retrying...",
|
|
415
|
+
self.name,
|
|
415
416
|
)
|
|
416
417
|
time.sleep(min(retries, self.MAX_RETRY_BACKOFF))
|
|
417
418
|
self.datastore.connection_reset()
|
|
@@ -470,7 +471,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
470
471
|
except elasticsearch.exceptions.TransportError as e:
|
|
471
472
|
err_code, _, _ = e.args
|
|
472
473
|
if err_code == 408 or err_code == "408":
|
|
473
|
-
logger.warning(
|
|
474
|
+
logger.warning("Waiting for index %s to get to status %s...", index, min_status)
|
|
474
475
|
else:
|
|
475
476
|
raise
|
|
476
477
|
|
|
@@ -576,8 +577,9 @@ class ESCollection(Generic[ModelType]):
|
|
|
576
577
|
|
|
577
578
|
if cur_shards > target_shards:
|
|
578
579
|
logger.info(
|
|
579
|
-
|
|
580
|
-
|
|
580
|
+
"Current shards (%s) is bigger then target shards (%s), we will be shrinking the index.",
|
|
581
|
+
cur_shards,
|
|
582
|
+
target_shards,
|
|
581
583
|
)
|
|
582
584
|
if cur_shards % target_shards != 0:
|
|
583
585
|
logger.info("The target shards is not a factor of the current shards, aborting...")
|
|
@@ -591,8 +593,9 @@ class ESCollection(Generic[ModelType]):
|
|
|
591
593
|
method = self.datastore.client.indices.shrink
|
|
592
594
|
elif cur_shards < target_shards:
|
|
593
595
|
logger.info(
|
|
594
|
-
|
|
595
|
-
|
|
596
|
+
"Current shards (%s) is smaller then target shards (%s), we will be splitting the index.",
|
|
597
|
+
cur_shards,
|
|
598
|
+
target_shards,
|
|
596
599
|
)
|
|
597
600
|
if target_shards % cur_shards != 0:
|
|
598
601
|
logger.warning("The current shards is not a factor of the target shards, aborting...")
|
|
@@ -601,13 +604,15 @@ class ESCollection(Generic[ModelType]):
|
|
|
601
604
|
method = self.datastore.client.indices.split
|
|
602
605
|
else:
|
|
603
606
|
logger.info(
|
|
604
|
-
|
|
605
|
-
"
|
|
607
|
+
"Current shards (%s) is equal to the target shards (%s), only housekeeping operations will be "
|
|
608
|
+
"performed.",
|
|
609
|
+
cur_shards,
|
|
610
|
+
target_shards,
|
|
606
611
|
)
|
|
607
612
|
|
|
608
613
|
if method:
|
|
609
614
|
# Before we do anything, we should make sure the source index is in a good state
|
|
610
|
-
logger.info(
|
|
615
|
+
logger.info("Waiting for %s status to be GREEN.", self.name.upper())
|
|
611
616
|
self._wait_for_status(self.name, min_status="green")
|
|
612
617
|
|
|
613
618
|
# Block all indexes to be written to
|
|
@@ -618,7 +623,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
618
623
|
if not self.with_retries(self.datastore.client.indices.exists, index=temp_name):
|
|
619
624
|
# if there are specific settings to be applied to the index, apply them
|
|
620
625
|
if clone_setup_settings:
|
|
621
|
-
logger.info(
|
|
626
|
+
logger.info("Relocating index to node %s.", target_node.upper())
|
|
622
627
|
self.with_retries(
|
|
623
628
|
self.datastore.client.indices.put_settings,
|
|
624
629
|
index=self.index_name,
|
|
@@ -630,7 +635,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
630
635
|
time.sleep(1)
|
|
631
636
|
|
|
632
637
|
# Make a clone of the current index
|
|
633
|
-
logger.info(
|
|
638
|
+
logger.info("Cloning %s into %s.", self.index_name.upper(), temp_name.upper())
|
|
634
639
|
self._safe_index_copy(
|
|
635
640
|
self.datastore.client.indices.clone,
|
|
636
641
|
self.index_name,
|
|
@@ -640,14 +645,16 @@ class ESCollection(Generic[ModelType]):
|
|
|
640
645
|
)
|
|
641
646
|
|
|
642
647
|
# Make 100% sure temporary index is ready
|
|
643
|
-
logger.info(
|
|
648
|
+
logger.info("Waiting for %s status to be GREEN.", temp_name.upper())
|
|
644
649
|
self._wait_for_status(temp_name, "green")
|
|
645
650
|
|
|
646
651
|
# Make sure temporary index is the alias if not already
|
|
647
652
|
if self._get_current_alias(self.name) != temp_name:
|
|
648
653
|
logger.info(
|
|
649
|
-
|
|
650
|
-
|
|
654
|
+
"Make %s the current alias for %s and delete %s.",
|
|
655
|
+
temp_name.upper(),
|
|
656
|
+
self.name.upper(),
|
|
657
|
+
self.index_name.upper(),
|
|
651
658
|
)
|
|
652
659
|
# Make the hot index the temporary index while deleting the original index
|
|
653
660
|
alias_actions = [
|
|
@@ -658,17 +665,19 @@ class ESCollection(Generic[ModelType]):
|
|
|
658
665
|
|
|
659
666
|
# Make sure the original index is deleted
|
|
660
667
|
if self.with_retries(self.datastore.client.indices.exists, index=self.index_name):
|
|
661
|
-
logger.info(
|
|
668
|
+
logger.info("Delete extra %s index.", self.index_name.upper())
|
|
662
669
|
self.with_retries(self.datastore.client.indices.delete, index=self.index_name)
|
|
663
670
|
|
|
664
671
|
# Shrink/split the temporary index into the original index
|
|
665
|
-
logger.info(
|
|
672
|
+
logger.info("Perform shard fix operation from %s to %s.", temp_name.upper(), self.index_name.upper())
|
|
666
673
|
self._safe_index_copy(method, temp_name, self.index_name, settings=settings)
|
|
667
674
|
|
|
668
675
|
# Make the original index the new alias
|
|
669
676
|
logger.info(
|
|
670
|
-
|
|
671
|
-
|
|
677
|
+
"Make %s the current alias for %s and delete %s.",
|
|
678
|
+
self.index_name.upper(),
|
|
679
|
+
self.name.upper(),
|
|
680
|
+
temp_name.upper(),
|
|
672
681
|
)
|
|
673
682
|
alias_actions = [
|
|
674
683
|
{"add": {"index": self.index_name, "alias": self.name}},
|
|
@@ -681,7 +690,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
681
690
|
self.with_retries(self.datastore.client.indices.put_settings, settings=write_unblock_settings)
|
|
682
691
|
|
|
683
692
|
# Restore normal routing and replicas
|
|
684
|
-
logger.debug(
|
|
693
|
+
logger.debug("Restore original routing table for %s.", self.name.upper())
|
|
685
694
|
self.with_retries(
|
|
686
695
|
self.datastore.client.indices.put_settings,
|
|
687
696
|
index=self.name,
|
|
@@ -849,7 +858,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
849
858
|
key_list.remove(row["_id"])
|
|
850
859
|
add_to_output(row["_source"], row["_id"])
|
|
851
860
|
except ValueError:
|
|
852
|
-
logger.exception(
|
|
861
|
+
logger.exception("MGet returned multiple documents for id: %s", row["_id"])
|
|
853
862
|
|
|
854
863
|
if key_list and error_on_missing:
|
|
855
864
|
raise MultiKeyError(key_list, out)
|
|
@@ -2358,7 +2367,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
2358
2367
|
"""
|
|
2359
2368
|
# Create HOT index
|
|
2360
2369
|
if not self.with_retries(self.datastore.client.indices.exists, index=self.name):
|
|
2361
|
-
logger.debug(
|
|
2370
|
+
logger.debug("Index %s does not exist. Creating it now...", self.name.upper())
|
|
2362
2371
|
try:
|
|
2363
2372
|
self.with_retries(
|
|
2364
2373
|
self.datastore.client.indices.create,
|
|
@@ -2369,7 +2378,7 @@ class ESCollection(Generic[ModelType]):
|
|
|
2369
2378
|
except elasticsearch.exceptions.RequestError as e:
|
|
2370
2379
|
if "resource_already_exists_exception" not in str(e):
|
|
2371
2380
|
raise
|
|
2372
|
-
logger.warning(
|
|
2381
|
+
logger.warning("Tried to create an index template that already exists: %s", self.name.upper())
|
|
2373
2382
|
|
|
2374
2383
|
self.with_retries(
|
|
2375
2384
|
self.datastore.client.indices.put_alias,
|
{howler_api-4.0.0.dev676 → howler_api-4.0.0.dev724}/howler/datastore/migrations/fix_process.py
RENAMED
|
@@ -15,12 +15,12 @@ def migrate():
|
|
|
15
15
|
logger.info("Preconditions met, continuing.")
|
|
16
16
|
|
|
17
17
|
db_size = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
|
|
18
|
-
logger.info(
|
|
18
|
+
logger.info("Database size pre-migration: %s", db_size)
|
|
19
19
|
else:
|
|
20
20
|
logger.info("Preconditions not met, stopping")
|
|
21
21
|
return
|
|
22
22
|
|
|
23
|
-
logger.info(
|
|
23
|
+
logger.info("We will delete %s hits. Continue?", result["total"])
|
|
24
24
|
prompt_result = input("y/[n]")
|
|
25
25
|
|
|
26
26
|
if prompt_result.lower() != "y":
|
|
@@ -32,7 +32,7 @@ def migrate():
|
|
|
32
32
|
collection.commit()
|
|
33
33
|
|
|
34
34
|
db_size_after = collection.search("howler.id:*", track_total_hits=True, rows=0)["total"]
|
|
35
|
-
logger.info(
|
|
35
|
+
logger.info("Database size post-migration: %s", db_size_after)
|
|
36
36
|
|
|
37
37
|
logger.info("Migration complete")
|
|
38
38
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# External scripts
|
|
2
|
+
|
|
3
|
+
## reindex_data.py
|
|
4
|
+
|
|
5
|
+
Reindex one or more Elasticsearch indexes used by Howler.
|
|
6
|
+
|
|
7
|
+
### Usage
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# Reindex specific indexes (confirms each one before proceeding)
|
|
11
|
+
python reindex_data.py hit user
|
|
12
|
+
|
|
13
|
+
# Reindex all indexes
|
|
14
|
+
python reindex_data.py --all
|
|
15
|
+
|
|
16
|
+
# Skip confirmation prompts and countdown
|
|
17
|
+
python reindex_data.py hit --force
|
|
18
|
+
|
|
19
|
+
# Print index schema before reindexing
|
|
20
|
+
python reindex_data.py hit --verbose
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### Options
|
|
24
|
+
|
|
25
|
+
| Argument | Description |
|
|
26
|
+
|-------------|----------------------------------------------|
|
|
27
|
+
| `indexes` | One or more index names to reindex. |
|
|
28
|
+
| `--all` | Reindex all indexes. |
|
|
29
|
+
| `--force` | Skip confirmation prompts and countdown. |
|
|
30
|
+
| `--verbose` | Print the index schema before reindexing. |
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
DELAY = 5
|
|
6
|
+
INDEX_NAMES = ["analytic", "hit", "view", "template", "overview", "action", "user", "dossier"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
if __name__ == "__main__":
|
|
10
|
+
parser = argparse.ArgumentParser(
|
|
11
|
+
description="Reindex elasticsearch indexes.",
|
|
12
|
+
epilog=f"Valid index names: {', '.join(INDEX_NAMES)}",
|
|
13
|
+
)
|
|
14
|
+
parser.add_argument("indexes", nargs="*", help="Indexes to reindex.")
|
|
15
|
+
parser.add_argument("--all", action="store_true", help="Reindex all indexes.")
|
|
16
|
+
parser.add_argument("--force", action="store_true", help="Skip confirmation prompts and countdown.")
|
|
17
|
+
parser.add_argument("--verbose", action="store_true", help="Print index schema before reindexing.")
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
|
|
20
|
+
if args.all and args.indexes:
|
|
21
|
+
parser.error("--all cannot be combined with positional index arguments.")
|
|
22
|
+
|
|
23
|
+
if not args.indexes and not args.all:
|
|
24
|
+
parser.error("Provide index names as arguments, or use --all.")
|
|
25
|
+
|
|
26
|
+
invalid = [name for name in args.indexes if name not in INDEX_NAMES]
|
|
27
|
+
if invalid:
|
|
28
|
+
parser.error(f"Invalid index(es): {', '.join(invalid)}. Valid options: {', '.join(INDEX_NAMES)}")
|
|
29
|
+
|
|
30
|
+
from howler.datastore.collection import ESCollection
|
|
31
|
+
|
|
32
|
+
ESCollection.IGNORE_ENSURE_COLLECTION = True
|
|
33
|
+
|
|
34
|
+
if args.force:
|
|
35
|
+
ESCollection.ENSURE_COLLECTION_WARNED = True
|
|
36
|
+
|
|
37
|
+
from howler.common import loader
|
|
38
|
+
|
|
39
|
+
ds = loader.datastore(archive_access=False)
|
|
40
|
+
|
|
41
|
+
selected = list(dict.fromkeys(INDEX_NAMES if args.all else args.indexes))
|
|
42
|
+
|
|
43
|
+
for index_name in selected:
|
|
44
|
+
collection: ESCollection = getattr(ds, index_name)
|
|
45
|
+
|
|
46
|
+
if args.verbose:
|
|
47
|
+
print(f"Index schema for '{index_name}':")
|
|
48
|
+
print(json.dumps(collection._get_index_mappings(), indent=2))
|
|
49
|
+
|
|
50
|
+
print(f"Reindexing: {', '.join(collection.index_list_full)}")
|
|
51
|
+
|
|
52
|
+
if not args.force:
|
|
53
|
+
answer = input(f"Are you sure you want to reindex '{index_name}'? [yes/NO] ")
|
|
54
|
+
if not answer.startswith("y"):
|
|
55
|
+
print("Confirmation not provided, skipping.")
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
for i in range(2 * DELAY):
|
|
59
|
+
print(f"Reindexing in {2 * DELAY - i}...", end="\r")
|
|
60
|
+
time.sleep(1)
|
|
61
|
+
print()
|
|
62
|
+
|
|
63
|
+
result = collection.reindex()
|
|
64
|
+
print(f"Reindex of '{index_name}' complete. Success: {result}.")
|
|
@@ -51,11 +51,11 @@ def get_apps_list(discovery_url: Optional[str]) -> list[dict[str, str]]:
|
|
|
51
51
|
)
|
|
52
52
|
|
|
53
53
|
except Exception:
|
|
54
|
-
logger.exception(
|
|
54
|
+
logger.exception("Failed to parse get app: %s", str(app))
|
|
55
55
|
else:
|
|
56
|
-
logger.warning(
|
|
56
|
+
logger.warning("Invalid response from server for apps discovery: %s", discovery_url)
|
|
57
57
|
except Exception:
|
|
58
|
-
logger.exception(
|
|
58
|
+
logger.exception("Failed to get apps from discover URL: %s", discovery_url)
|
|
59
59
|
|
|
60
60
|
DISCO_CACHE[discovery_url] = sorted(apps, key=lambda k: k["name"])
|
|
61
61
|
return sorted(apps, key=lambda k: k["name"])
|
|
@@ -1503,11 +1503,31 @@ def recursive_set_name(field, name, to_parent=False):
|
|
|
1503
1503
|
recursive_set_name(field.child_type, name, to_parent=True)
|
|
1504
1504
|
|
|
1505
1505
|
|
|
1506
|
-
def model(index=None, store=None, description=None):
|
|
1507
|
-
"""Decorator
|
|
1506
|
+
def model(index=None, store=None, description=None, id_field=None):
|
|
1507
|
+
"""Decorator that finalizes a Model subclass for use with the datastore.
|
|
1508
|
+
Assigns metadata to the class (description, id field), validates that all
|
|
1509
|
+
declared field names are legal, recursively sets each field's name, and
|
|
1510
|
+
applies default index/store settings to every field.
|
|
1511
|
+
If ``id_field`` is not provided, it defaults to ``<classname_lower>_id``.
|
|
1512
|
+
Args:
|
|
1513
|
+
index: Default index setting applied to all fields on the model.
|
|
1514
|
+
store: Default store setting applied to all fields on the model.
|
|
1515
|
+
description: Human-readable description of the model.
|
|
1516
|
+
id_field: Name of the field used as the primary key. Defaults to
|
|
1517
|
+
``<classname_lower>_id`` when not specified.
|
|
1518
|
+
Returns:
|
|
1519
|
+
A class decorator that configures and returns the decorated Model subclass.
|
|
1520
|
+
Raises:
|
|
1521
|
+
HowlerValueError: If any field name fails the ``FIELD_SANITIZER`` regex
|
|
1522
|
+
or appears in ``BANNED_FIELDS``.
|
|
1523
|
+
"""
|
|
1508
1524
|
|
|
1509
1525
|
def _finish_model(cls):
|
|
1510
1526
|
cls._Model__description = description
|
|
1527
|
+
cls._Model__id_field = id_field
|
|
1528
|
+
|
|
1529
|
+
if cls._Model__id_field is None:
|
|
1530
|
+
cls._Model__id_field = f"{cls.__name__.lower()}_id"
|
|
1511
1531
|
|
|
1512
1532
|
for name, field_data in cls.fields().items():
|
|
1513
1533
|
if not FIELD_SANITIZER.match(name) or name in BANNED_FIELDS:
|
|
@@ -471,7 +471,7 @@ def create_users_with_username(ds: HowlerDatastore, usernames: list[str]):
|
|
|
471
471
|
ds.user.save(username, user_data)
|
|
472
472
|
|
|
473
473
|
if "pytest" not in sys.modules:
|
|
474
|
-
logger.info(
|
|
474
|
+
logger.info("%s:%s", username, username)
|
|
475
475
|
|
|
476
476
|
ds.user.commit()
|
|
477
477
|
ds.user_avatar.commit()
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Datastore convenience mixins for Howler ODM Model classes.
|
|
2
|
+
|
|
3
|
+
Provides :class:`DatastoreMixin`, a generic mixin that adds a class-level
|
|
4
|
+
``store`` property (returning a typed :class:`ESCollection`) and instance-level
|
|
5
|
+
``ds`` / ``save`` helpers so that Model subclasses can interact with the
|
|
6
|
+
Elasticsearch datastore without boilerplate.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from operator import attrgetter
|
|
12
|
+
from typing import Generic, TypeVar, overload
|
|
13
|
+
|
|
14
|
+
from howler.common.exceptions import HowlerRuntimeError
|
|
15
|
+
from howler.common.loader import datastore
|
|
16
|
+
from howler.datastore.collection import ESCollection
|
|
17
|
+
from howler.odm.base import Model
|
|
18
|
+
|
|
19
|
+
ModelType = TypeVar("ModelType", bound=Model)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _ObjectsDescriptor(Generic[ModelType]):
|
|
23
|
+
"""Descriptor that provides class-level-only access to the model's ESCollection.
|
|
24
|
+
|
|
25
|
+
Intended to be accessed exclusively via the class (e.g. ``Case.store``).
|
|
26
|
+
Raises ``AttributeError`` if accessed from an instance to enforce
|
|
27
|
+
cla # noqa: D205
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
@overload
|
|
31
|
+
def __get__(self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
32
|
+
|
|
33
|
+
@overload
|
|
34
|
+
def __get__(self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
|
|
35
|
+
|
|
36
|
+
def __get__(self, obj: ModelType | None, objtype: type[ModelType] | None = None) -> ESCollection[ModelType]:
|
|
37
|
+
"""Return the ESCollection for the owner class.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
obj: The instance the descriptor was accessed from, or ``None``
|
|
41
|
+
when accessed via the class.
|
|
42
|
+
objtype: The owner class (e.g. ``Case``, ``Hit``).
|
|
43
|
+
Returns:
|
|
44
|
+
ESCollection[ModelType]: The datastore collection for *objtype*.
|
|
45
|
+
Raises:
|
|
46
|
+
AttributeError: If accessed from an instance (*obj* is not ``None``)
|
|
47
|
+
or if *objtype* cannot be determined.
|
|
48
|
+
"""
|
|
49
|
+
if obj is not None:
|
|
50
|
+
raise HowlerRuntimeError(
|
|
51
|
+
f"'{type(obj).__name__}.store' is a class-level property and cannot be accessed from an instance. "
|
|
52
|
+
f"Use '{type(obj).__name__}.store' instead."
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
if objtype is None:
|
|
56
|
+
raise HowlerRuntimeError("Cannot resolve owner class for 'store' descriptor.")
|
|
57
|
+
|
|
58
|
+
index_name = objtype.__name__.lower()
|
|
59
|
+
return datastore()[index_name]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class DatastoreMixin(Generic[ModelType]):
|
|
63
|
+
"""Mixin that provides convenience datastore access to Model instances.
|
|
64
|
+
|
|
65
|
+
Generic over ``ModelType`` so that the ``store`` class property returns a
|
|
66
|
+
correctly-typed ``ESCollection[ModelType]``. Adds a ``ds`` property for
|
|
67
|
+
retrieving the shared datastore connection, a ``store`` class-only property
|
|
68
|
+
for retrieving the model's ESCollection (raises ``AttributeError`` if accessed
|
|
69
|
+
from an instance), and a ``save`` method that persists the current model
|
|
70
|
+
instance using its class name as the index and its configured ID field as the
|
|
71
|
+
document key.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
store: _ObjectsDescriptor = _ObjectsDescriptor()
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def ds(self):
|
|
78
|
+
"""Return the shared datastore instance.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
The singleton datastore connection used for all persistence operations.
|
|
82
|
+
"""
|
|
83
|
+
return datastore()
|
|
84
|
+
|
|
85
|
+
def save(self) -> bool:
|
|
86
|
+
"""Persist the current model instance to the datastore.
|
|
87
|
+
|
|
88
|
+
Determines the target index from the lowercase class name, extracts the
|
|
89
|
+
model's ID from the configured ID field, and saves the instance.
|
|
90
|
+
Returns:
|
|
91
|
+
bool: True if the save operation succeeded, False otherwise.
|
|
92
|
+
"""
|
|
93
|
+
index_name = self.__class__.__name__.lower()
|
|
94
|
+
id_field = self.__class__._Model__id_field # type: ignore[attr-defined]
|
|
95
|
+
current_id = attrgetter(id_field)(self)
|
|
96
|
+
|
|
97
|
+
return self.ds[index_name].save(current_id, self)
|
|
@@ -3,6 +3,7 @@ from typing import Any, Optional
|
|
|
3
3
|
from howler import odm
|
|
4
4
|
from howler.common.exceptions import HowlerValueError
|
|
5
5
|
from howler.odm.constants import Status
|
|
6
|
+
from howler.odm.mixins import DatastoreMixin
|
|
6
7
|
from howler.utils.compat import StrEnum
|
|
7
8
|
|
|
8
9
|
CASE_ITEM_TYPES = {"observable", "hit", "case", "lead", "reference"}
|
|
@@ -92,7 +93,7 @@ class CaseEnrichment(odm.Model):
|
|
|
92
93
|
|
|
93
94
|
|
|
94
95
|
@odm.model(index=True, store=True, description="Case model with path-based items, enrichments, rules, and tasks.")
|
|
95
|
-
class Case(odm.Model):
|
|
96
|
+
class Case(DatastoreMixin["Case"], odm.Model):
|
|
96
97
|
case_id: str = odm.UUID(description="A unique identifier for this case.")
|
|
97
98
|
title: str = odm.Keyword(description="Case title.")
|
|
98
99
|
summary: str = odm.Text(description="Short case summary.")
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from howler import odm
|
|
4
4
|
from howler.common.logging import get_logger
|
|
5
|
+
from howler.odm.mixins import DatastoreMixin
|
|
5
6
|
from howler.odm.models.howler_data import HowlerData
|
|
6
7
|
from howler.odm.models.record import Record
|
|
7
8
|
|
|
@@ -12,8 +13,9 @@ logger = get_logger(__file__)
|
|
|
12
13
|
index=True,
|
|
13
14
|
store=True,
|
|
14
15
|
description="Howler Outline schema which is an extended version of Elastic Common Schema (ECS)",
|
|
16
|
+
id_field="howler.id",
|
|
15
17
|
)
|
|
16
|
-
class Hit(Record):
|
|
18
|
+
class Hit(DatastoreMixin["Hit"], Record):
|
|
17
19
|
# Howler extended fields. Deviates from ECS
|
|
18
20
|
howler: HowlerData = odm.Compound(
|
|
19
21
|
HowlerData,
|
|
@@ -4,6 +4,7 @@ from howler import odm
|
|
|
4
4
|
from howler.common.exceptions import HowlerValueError
|
|
5
5
|
from howler.common.logging import get_logger
|
|
6
6
|
from howler.odm.howler_enum import HowlerEnum
|
|
7
|
+
from howler.odm.mixins import DatastoreMixin
|
|
7
8
|
from howler.odm.models.record import Record
|
|
8
9
|
|
|
9
10
|
logger = get_logger(__file__)
|
|
@@ -110,8 +111,9 @@ class ObservableData(odm.Model):
|
|
|
110
111
|
index=True,
|
|
111
112
|
store=True,
|
|
112
113
|
description="Observable schema which is an extended version of Elastic Common Schema (ECS)",
|
|
114
|
+
id_field="howler.id",
|
|
113
115
|
)
|
|
114
|
-
class Observable(Record):
|
|
116
|
+
class Observable(DatastoreMixin["Observable"], Record):
|
|
115
117
|
# Howler extended fields. Deviates from ECS
|
|
116
118
|
howler: ObservableData = odm.Compound(
|
|
117
119
|
ObservableData,
|