howler-api 4.0.0.dev973__tar.gz → 4.0.0.dev982__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/PKG-INFO +1 -1
  2. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/demote.py +2 -1
  3. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/promote.py +6 -1
  4. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/collection.py +244 -61
  5. howler_api-4.0.0.dev982/howler/external/reindex_data.py +134 -0
  6. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/hit.py +10 -1
  7. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/helper.py +1 -0
  8. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/howler_data.py +6 -0
  9. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/random_data.py +1 -0
  10. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/hit_service.py +1 -1
  11. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/pyproject.toml +1 -1
  12. howler_api-4.0.0.dev973/howler/external/reindex_data.py +0 -64
  13. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/README.md +0 -0
  14. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/__init__.py +0 -0
  15. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/__init__.py +0 -0
  16. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/add_label.py +0 -0
  17. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/add_to_bundle.py +0 -0
  18. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/change_field.py +0 -0
  19. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/example_plugin.py +0 -0
  20. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/prioritization.py +0 -0
  21. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/remove_from_bundle.py +0 -0
  22. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/remove_label.py +0 -0
  23. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/actions/transition.py +0 -0
  24. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/__init__.py +0 -0
  25. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/base.py +0 -0
  26. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/socket.py +0 -0
  27. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/__init__.py +0 -0
  28. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/action.py +0 -0
  29. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/analytic.py +0 -0
  30. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/auth.py +0 -0
  31. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/clue.py +0 -0
  32. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/configs.py +0 -0
  33. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/dossier.py +0 -0
  34. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/help.py +0 -0
  35. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/hit.py +0 -0
  36. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/notebook.py +0 -0
  37. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/overview.py +0 -0
  38. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/search.py +0 -0
  39. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/template.py +0 -0
  40. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/tool.py +0 -0
  41. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/user.py +0 -0
  42. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/utils/__init__.py +0 -0
  43. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/utils/etag.py +0 -0
  44. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/api/v1/view.py +0 -0
  45. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/app.py +0 -0
  46. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/README.md +0 -0
  47. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/__init__.py +0 -0
  48. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/classification.py +0 -0
  49. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/classification.yml +0 -0
  50. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/exceptions.py +0 -0
  51. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/loader.py +0 -0
  52. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/__init__.py +0 -0
  53. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/audit.py +0 -0
  54. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/logging/format.py +0 -0
  55. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/net.py +0 -0
  56. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/net_static.py +0 -0
  57. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/random_user.py +0 -0
  58. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/common/swagger.py +0 -0
  59. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/config.py +0 -0
  60. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/__init__.py +0 -0
  61. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/action_queue_worker.py +0 -0
  62. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/retention.py +0 -0
  63. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/rules.py +0 -0
  64. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/cronjobs/view_cleanup.py +0 -0
  65. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/README.md +0 -0
  66. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/__init__.py +0 -0
  67. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/bulk.py +0 -0
  68. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/constants.py +0 -0
  69. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/exceptions.py +0 -0
  70. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/howler_store.py +0 -0
  71. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/migrations/fix_process.py +0 -0
  72. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/operations.py +0 -0
  73. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/schemas.py +0 -0
  74. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/store.py +0 -0
  75. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/__init__.py +0 -0
  76. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/build.py +0 -0
  77. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/support/schemas.py +0 -0
  78. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/datastore/types.py +0 -0
  79. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/error.py +0 -0
  80. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/README.md +0 -0
  81. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/__init__.py +0 -0
  82. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_mitre.py +0 -0
  83. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_sigma_rules.py +0 -0
  84. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/generate_tlds.py +0 -0
  85. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/external/wipe_databases.py +0 -0
  86. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/gunicorn_config.py +0 -0
  87. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/healthz.py +0 -0
  88. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/__init__.py +0 -0
  89. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/azure.py +0 -0
  90. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/discover.py +0 -0
  91. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/oauth.py +0 -0
  92. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/search.py +0 -0
  93. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/workflow.py +0 -0
  94. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/helper/ws.py +0 -0
  95. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/README.md +0 -0
  96. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/__init__.py +0 -0
  97. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/base.py +0 -0
  98. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/charter.txt +0 -0
  99. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/howler_enum.py +0 -0
  100. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/__init__.py +0 -0
  101. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/action.py +0 -0
  102. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/analytic.py +0 -0
  103. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/assemblyline.py +0 -0
  104. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/aws.py +0 -0
  105. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/azure.py +0 -0
  106. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/cbs.py +0 -0
  107. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/clue.py +0 -0
  108. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/config.py +0 -0
  109. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/dossier.py +0 -0
  110. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/__init__.py +0 -0
  111. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/agent.py +0 -0
  112. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/autonomous_system.py +0 -0
  113. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/client.py +0 -0
  114. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/cloud.py +0 -0
  115. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/code_signature.py +0 -0
  116. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/container.py +0 -0
  117. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/dns.py +0 -0
  118. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/egress.py +0 -0
  119. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/elf.py +0 -0
  120. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/email.py +0 -0
  121. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/error.py +0 -0
  122. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/event.py +0 -0
  123. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/faas.py +0 -0
  124. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/file.py +0 -0
  125. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/geo.py +0 -0
  126. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/group.py +0 -0
  127. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/hash.py +0 -0
  128. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/host.py +0 -0
  129. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/http.py +0 -0
  130. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/ingress.py +0 -0
  131. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/interface.py +0 -0
  132. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/network.py +0 -0
  133. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/observer.py +0 -0
  134. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/organization.py +0 -0
  135. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/os.py +0 -0
  136. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/pe.py +0 -0
  137. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/process.py +0 -0
  138. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/registry.py +0 -0
  139. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/related.py +0 -0
  140. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/rule.py +0 -0
  141. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/server.py +0 -0
  142. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/threat.py +0 -0
  143. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/tls.py +0 -0
  144. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/url.py +0 -0
  145. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/user.py +0 -0
  146. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/user_agent.py +0 -0
  147. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/ecs/vulnerability.py +0 -0
  148. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/gcp.py +0 -0
  149. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/hit.py +0 -0
  150. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/lead.py +0 -0
  151. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/localized_label.py +0 -0
  152. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/overview.py +0 -0
  153. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/pivot.py +0 -0
  154. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/template.py +0 -0
  155. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/user.py +0 -0
  156. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/models/view.py +0 -0
  157. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/odm/randomizer.py +0 -0
  158. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/patched.py +0 -0
  159. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/plugins/__init__.py +0 -0
  160. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/plugins/config.py +0 -0
  161. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/__init__.py +0 -0
  162. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/README.md +0 -0
  163. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/__init__.py +0 -0
  164. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/counters.py +0 -0
  165. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/events.py +0 -0
  166. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/hash.py +0 -0
  167. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/lock.py +0 -0
  168. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/__init__.py +0 -0
  169. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/comms.py +0 -0
  170. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/multi.py +0 -0
  171. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/named.py +0 -0
  172. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/queues/priority.py +0 -0
  173. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/set.py +0 -0
  174. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/remote/datatypes/user_quota_tracker.py +0 -0
  175. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/__init__.py +0 -0
  176. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/socket.py +0 -0
  177. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/security/utils.py +0 -0
  178. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/__init__.py +0 -0
  179. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/action_service.py +0 -0
  180. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/analytic_service.py +0 -0
  181. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/auth_service.py +0 -0
  182. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/config_service.py +0 -0
  183. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/dossier_service.py +0 -0
  184. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/event_service.py +0 -0
  185. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/jwt_service.py +0 -0
  186. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/lucene_service.py +0 -0
  187. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/notebook_service.py +0 -0
  188. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/overview_service.py +0 -0
  189. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/template_service.py +0 -0
  190. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/services/user_service.py +0 -0
  191. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/telemetry.py +0 -0
  192. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/__init__.py +0 -0
  193. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/annotations.py +0 -0
  194. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/chunk.py +0 -0
  195. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/compat.py +0 -0
  196. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/constants.py +0 -0
  197. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/dict_utils.py +0 -0
  198. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/isotime.py +0 -0
  199. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/list_utils.py +0 -0
  200. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/lucene.py +0 -0
  201. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/path.py +0 -0
  202. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/socket_utils.py +0 -0
  203. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/str_utils.py +0 -0
  204. {howler_api-4.0.0.dev973 → howler_api-4.0.0.dev982}/howler/utils/uid.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: howler-api
3
- Version: 4.0.0.dev973
3
+ Version: 4.0.0.dev982
4
4
  Summary: Howler - API server
5
5
  License: MIT
6
6
  Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -76,6 +76,7 @@ def execute(
76
76
  *hit_helper.demote_hit(escalation=escalation),
77
77
  odm_helper.update("howler.assessment", None),
78
78
  odm_helper.update("howler.rationale", None),
79
+ odm_helper.update("howler.assignment", None),
79
80
  ],
80
81
  )
81
82
  else:
@@ -93,7 +94,7 @@ def execute(
93
94
  ds.hit.update_by_query(
94
95
  query,
95
96
  [
96
- *hit_helper.assess_hit(assessment, rationale),
97
+ *hit_helper.assess_hit(assessment, rationale, user=(user if user else "automation")),
97
98
  odm_helper.update(
98
99
  "howler.assignment",
99
100
  user.get("uname", "automation") if user else "automation",
@@ -10,6 +10,7 @@ from howler.odm.models.howler_data import (
10
10
  AssessmentEscalationMap,
11
11
  Escalation,
12
12
  )
13
+ from howler.odm.models.user import User
13
14
  from howler.utils.str_utils import sanitize_lucene_query
14
15
 
15
16
  OPERATION_ID = "promote"
@@ -26,6 +27,7 @@ def execute(
26
27
  escalation: Escalation = Escalation.ALERT,
27
28
  assessment: Optional[str] = None,
28
29
  rationale: Optional[str] = None,
30
+ user: Optional[User] = None,
29
31
  **kwargs,
30
32
  ):
31
33
  """Promote a hit.
@@ -73,6 +75,7 @@ def execute(
73
75
  *hit_helper.promote_hit(escalation=escalation),
74
76
  odm_helper.update("howler.assessment", None),
75
77
  odm_helper.update("howler.rationale", None),
78
+ odm_helper.update("howler.assignment", None),
76
79
  ],
77
80
  )
78
81
  else:
@@ -87,7 +90,9 @@ def execute(
87
90
  )
88
91
  return report
89
92
 
90
- ds.hit.update_by_query(query, hit_helper.assess_hit(assessment, rationale))
93
+ ds.hit.update_by_query(
94
+ query, hit_helper.assess_hit(assessment, rationale, user=(user if user else "automation"))
95
+ )
91
96
 
92
97
  report.append(
93
98
  {
@@ -480,8 +480,8 @@ class ESCollection(Generic[ModelType]):
480
480
  else:
481
481
  raise
482
482
 
483
- def _safe_index_copy(self, copy_function, src, target, settings=None, min_status="yellow"):
484
- options_client = self.datastore.client.options(request_timeout=60)
483
+ def _safe_index_copy(self, copy_function, src, target, settings=None, min_status="yellow", request_timeout=60):
484
+ options_client = self.datastore.client.options(request_timeout=request_timeout)
485
485
  timed_function = getattr(options_client.indices, copy_function.__name__)
486
486
  ret = timed_function(index=src, target=target, settings=settings)
487
487
  if not ret["acknowledged"]:
@@ -703,22 +703,80 @@ class ESCollection(Generic[ModelType]):
703
703
  settings=clone_finish_settings,
704
704
  )
705
705
 
706
- def reindex(self):
707
- """This function should be overloaded to perform a reindex of all the data of the different hosts
708
- specified in self.datastore.hosts.
706
+ def _index_doc_count(self, index: str) -> int:
707
+ """Return the number of documents in a physical index.
709
708
 
710
- :return: Should return True of the commit was successful on all hosts
709
+ :param index: the name of the physical index to count documents in
710
+ :return: the number of documents currently stored in the index
711
+ """
712
+ self.with_retries(self.datastore.client.indices.refresh, index=index)
713
+ return self.with_retries(self.datastore.client.count, index=index)["count"]
714
+
715
+ def reindex(self, allow_failures: bool = False, request_timeout: int = 60):
716
+ """Reindex all the data of the collection into a freshly mapped index.
717
+
718
+ For every index in ``self.index_list`` the data is copied into a temporary
719
+ ``__reindex`` index that uses the current mappings/settings. Writes to the source
720
+ index are blocked while the copy runs so the reindex result and document counts can
721
+ be validated before the original index is deleted. The temporary index is only
722
+ collapsed back onto the original name once those checks pass.
723
+
724
+ :param allow_failures: when ``True``, proceed even if the reindex reported document
725
+ failures or version conflicts, or if the document counts do not match. This is
726
+ DESTRUCTIVE: documents that could not be converted to the new mappings will be
727
+ permanently dropped. Only use this for intentional lossy migrations.
728
+ :param request_timeout: transport timeout in seconds for synchronous index-copy
729
+ operations. Defaults to 60 seconds.
730
+ :return: ``True`` when the reindex (and validation) completed successfully on all
731
+ indexes.
732
+ :raises DataStoreException: if a reindex reported failures/conflicts, or if the
733
+ document count of the reindexed data does not match the source, and
734
+ ``allow_failures`` is ``False``. The ``__reindex`` index is left in place so the
735
+ operation can be recovered with :meth:`reindex_cleanup`.
711
736
  """
712
737
  logger.warning("Beginning Reindex")
713
738
  for index in self.index_list:
714
739
  new_name = f"{index}__reindex"
715
740
  index_data = None
716
- if self.with_retries(self.datastore.client.indices.exists, index=index) and not self.with_retries(
717
- self.datastore.client.indices.exists, index=new_name
718
- ):
741
+ source_count = None
742
+ source_writes_blocked = False
743
+
744
+ source_exists = self.with_retries(self.datastore.client.indices.exists, index=index)
745
+ target_exists = self.with_retries(self.datastore.client.indices.exists, index=new_name)
746
+
747
+ # Never reindex while a '__reindex' index already exists. Its presence means a previous
748
+ # reindex was interrupted, and the leftover may be stale or incomplete. Committing it
749
+ # could silently replace live data, so force the operator to reconcile the state with
750
+ # --cleanup before any new reindex is attempted.
751
+ if target_exists:
752
+ raise DataStoreException(
753
+ f"A leftover reindex index '{new_name}' already exists. This usually means a previous "
754
+ f"reindex was interrupted. Refusing to reindex because '{new_name}' may contain stale "
755
+ f"or incomplete data. Run the reindex script with --cleanup to reconcile the state "
756
+ f"if '{index}' still exists, or manually recover '{new_name}' if the source index is "
757
+ f"missing, then retry the reindex."
758
+ )
759
+
760
+ if not source_exists:
761
+ logger.warning("Neither %s nor %s exist, nothing to reindex.", index, new_name)
762
+ continue
763
+
764
+ try:
719
765
  # Get information about the index to reindex
720
766
  index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
721
767
 
768
+ logger.warning("Block writes to source index %s", index)
769
+ self.with_retries(
770
+ self.datastore.client.indices.put_settings,
771
+ index=index,
772
+ settings=write_block_settings,
773
+ )
774
+ source_writes_blocked = True
775
+
776
+ # Record the number of documents we expect to migrate after writes are blocked.
777
+ source_count = self._index_doc_count(index)
778
+ logger.warning("Source index %s contains %s document(s)", index, source_count)
779
+
722
780
  # Create reindex target
723
781
  logger.warning("Creating new index with name %s", new_name)
724
782
  self.with_retries(
@@ -728,32 +786,6 @@ class ESCollection(Generic[ModelType]):
728
786
  settings=self._get_index_settings(),
729
787
  )
730
788
 
731
- # For all aliases related to the index, add a new alias to the reindex index
732
- for alias, alias_data in index_data["aliases"].items():
733
- # Make the reindex index the new write index if the original index was
734
- if alias_data.get("is_write_index", True):
735
- alias_actions = [
736
- {
737
- "add": {
738
- "index": new_name,
739
- "alias": alias,
740
- "is_write_index": True,
741
- }
742
- },
743
- {
744
- "add": {
745
- "index": index,
746
- "alias": alias,
747
- "is_write_index": False,
748
- }
749
- },
750
- ]
751
- else:
752
- alias_actions = [{"add": {"index": new_name, "alias": alias}}]
753
-
754
- logger.warning("Updating alias %s", alias)
755
- self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
756
-
757
789
  # Reindex data into target
758
790
  logger.warning("Beginning reindex from %s to %s", index, new_name)
759
791
  r_task = self.with_retries(
@@ -763,23 +795,29 @@ class ESCollection(Generic[ModelType]):
763
795
  wait_for_completion=False,
764
796
  )
765
797
  logger.warning("Reindex taskId: %s", r_task["task"])
766
- self._get_task_results(r_task)
798
+ reindex_result = self._get_task_results(r_task)
767
799
 
768
- if self.with_retries(self.datastore.client.indices.exists, index=new_name):
769
- if index_data is None:
770
- index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
800
+ # Validate the reindex did not silently drop or conflict on documents before
801
+ # we commit to deleting the source index further down.
802
+ self._validate_reindex_result(index, new_name, reindex_result, allow_failures)
771
803
 
772
804
  logger.warning("Committing reindexed data in index %s", new_name)
773
805
  self.with_retries(self.datastore.client.indices.refresh, index=new_name)
774
806
  self.with_retries(self.datastore.client.indices.clear_cache, index=new_name)
775
807
 
808
+ # Compare the document counts of the source and reindexed indexes before deleting
809
+ # the source so a silent document drop is caught.
810
+ target_count = self._index_doc_count(new_name)
811
+ self._validate_reindex_counts(index, new_name, source_count, target_count, allow_failures)
812
+
776
813
  logger.warning("Deleting old index %s", index)
777
- if self.with_retries(self.datastore.client.indices.exists, index=index):
778
- self.with_retries(self.datastore.client.indices.delete, index=index)
814
+ self.with_retries(self.datastore.client.indices.delete, index=index)
815
+ source_writes_blocked = False
779
816
 
780
- logger.warning("Block write to index")
817
+ logger.warning("Block writes to reindex target %s", new_name)
781
818
  self.with_retries(
782
819
  self.datastore.client.indices.put_settings,
820
+ index=new_name,
783
821
  settings=write_block_settings,
784
822
  )
785
823
 
@@ -790,36 +828,181 @@ class ESCollection(Generic[ModelType]):
790
828
  new_name,
791
829
  index,
792
830
  settings=self._get_index_settings(),
831
+ request_timeout=request_timeout,
793
832
  )
794
833
 
795
- # Restore original aliases for the index
796
- for alias, alias_data in index_data["aliases"].items():
797
- # Make the reindex index the new write index if the original index was
798
- if alias_data.get("is_write_index", True):
799
- alias_actions = [
800
- {
801
- "add": {
802
- "index": index,
803
- "alias": alias,
804
- "is_write_index": True,
805
- }
806
- },
807
- {"remove_index": {"index": new_name}},
808
- ]
809
- self.with_retries(
810
- self.datastore.client.indices.update_aliases,
811
- actions=alias_actions,
812
- )
834
+ alias_actions = []
835
+ aliases = index_data.get("aliases", {}) or {self.name: {"is_write_index": True}}
836
+ for alias, alias_data in aliases.items():
837
+ alias_action = {"index": index, "alias": alias}
838
+ alias_action.update(alias_data)
839
+ if alias == self.name or alias_data.get("is_write_index", False):
840
+ alias_action["is_write_index"] = True
841
+ alias_actions.append({"add": alias_action})
842
+ alias_actions.append({"remove_index": {"index": new_name}})
843
+ self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
813
844
 
814
845
  if self.with_retries(self.datastore.client.indices.exists, index=new_name):
815
846
  logger.warning("Deleting reindex target %s", new_name)
816
847
  self.with_retries(self.datastore.client.indices.delete, index=new_name)
817
848
  finally:
818
- logger.warning("Unblock write to the index")
849
+ if self.with_retries(self.datastore.client.indices.exists, index=index):
850
+ logger.warning("Unblock writes to the index")
851
+ self.with_retries(
852
+ self.datastore.client.indices.put_settings,
853
+ index=index,
854
+ settings=write_unblock_settings,
855
+ )
856
+ except Exception:
857
+ if source_writes_blocked and self.with_retries(self.datastore.client.indices.exists, index=index):
858
+ logger.warning("Unblock writes to source index %s after failed reindex", index)
819
859
  self.with_retries(
820
860
  self.datastore.client.indices.put_settings,
861
+ index=index,
821
862
  settings=write_unblock_settings,
822
863
  )
864
+ raise
865
+
866
+ return True
867
+
868
+ def _validate_reindex_result(self, index, new_name, reindex_result, allow_failures):
869
+ """Validate the result of an Elasticsearch reindex task.
870
+
871
+ :param index: the source index being reindexed
872
+ :param new_name: the temporary ``__reindex`` index being written to
873
+ :param reindex_result: the ``response``/``status`` payload returned by the reindex task
874
+ :param allow_failures: when ``True``, log the problems but do not abort
875
+ :raises DataStoreException: if the reindex reported failures or version conflicts and
876
+ ``allow_failures`` is ``False``
877
+ """
878
+ failures = reindex_result.get("failures", []) if reindex_result else []
879
+ version_conflicts = reindex_result.get("version_conflicts", 0) if reindex_result else 0
880
+
881
+ if not failures and not version_conflicts:
882
+ return
883
+
884
+ # Summarize the failures so the operator understands what went wrong
885
+ summary = (
886
+ f"Reindex of {index} into {new_name} reported {len(failures)} document failure(s) "
887
+ f"and {version_conflicts} version conflict(s)."
888
+ )
889
+ for failure in failures[:10]:
890
+ cause = failure.get("cause", failure)
891
+ logger.error(
892
+ "Reindex failure on document %s: %s - %s",
893
+ failure.get("id", "<unknown>"),
894
+ cause.get("type", "<unknown>"),
895
+ cause.get("reason", cause),
896
+ )
897
+ if len(failures) > 10:
898
+ logger.error("... and %s additional failure(s) not shown.", len(failures) - 10)
899
+
900
+ if allow_failures:
901
+ logger.warning("%s Proceeding anyway because allow_failures is set (DESTRUCTIVE).", summary)
902
+ return
903
+
904
+ raise DataStoreException(
905
+ f"{summary} Aborting before deleting the source index to prevent data loss. The '{new_name}' "
906
+ f"index has been left in place; run the reindex script with --cleanup to remove it and restore "
907
+ f"'{index}', then retry. Re-run with --allow-failures only if you intend to drop the "
908
+ f"un-convertible documents."
909
+ )
910
+
911
+ def _validate_reindex_counts(self, index, new_name, source_count, target_count, allow_failures):
912
+ """Validate that the reindexed index contains the same number of documents as the source.
913
+
914
+ :param index: the source index being reindexed
915
+ :param new_name: the temporary ``__reindex`` index being written to
916
+ :param source_count: the number of documents in the source index
917
+ :param target_count: the number of documents in the reindexed index
918
+ :param allow_failures: when ``True``, log the mismatch but do not abort
919
+ :raises DataStoreException: if the counts differ and ``allow_failures`` is ``False``
920
+ """
921
+ if source_count == target_count:
922
+ logger.warning("Document count validated: %s document(s) in both %s and %s", source_count, index, new_name)
923
+ return
924
+
925
+ summary = (
926
+ f"Document count mismatch reindexing {index} into {new_name}: "
927
+ f"source has {source_count} document(s) but reindex target has {target_count}."
928
+ )
929
+
930
+ if allow_failures:
931
+ logger.warning("%s Proceeding anyway because allow_failures is set (DESTRUCTIVE).", summary)
932
+ return
933
+
934
+ raise DataStoreException(
935
+ f"{summary} Aborting before deleting the source index to prevent data loss. The '{new_name}' "
936
+ f"index has been left in place; run the reindex script with --cleanup to remove it and restore "
937
+ f"'{index}', then retry. Re-run with --allow-failures only if you intend to accept the count "
938
+ f"mismatch."
939
+ )
940
+
941
+ def reindex_cleanup(self):
942
+ """Recover from a failed or interrupted :meth:`reindex` run.
943
+
944
+ For every index that still has a leftover ``__reindex`` index, restore the source
945
+ index as the writable target and delete the orphaned ``__reindex`` index. If the
946
+ source index is missing, raise an error instead of deleting ``__reindex`` because
947
+ it may be the only remaining copy of the data.
948
+
949
+ Write blocks set during reindexing are cleared so the collection is usable again.
950
+
951
+ :return: ``True`` when cleanup completed on all indexes
952
+ """
953
+ logger.warning("Beginning reindex cleanup")
954
+ for index in self.index_list:
955
+ new_name = f"{index}__reindex"
956
+
957
+ if not self.with_retries(self.datastore.client.indices.exists, index=new_name):
958
+ logger.info("No leftover reindex index found for %s, nothing to clean up.", index)
959
+ continue
960
+
961
+ if not self.with_retries(self.datastore.client.indices.exists, index=index):
962
+ raise DataStoreException(
963
+ f"Source index '{index}' is missing but leftover reindex index '{new_name}' exists. "
964
+ f"Cannot safely clean up because '{new_name}' may be the only remaining copy of the data. "
965
+ f"Manually recover '{new_name}' or delete it if the data is no longer needed."
966
+ )
967
+
968
+ logger.warning("Restoring aliases for %s and removing leftover index %s", index, new_name)
969
+ index_data = self.with_retries(self.datastore.client.indices.get, index=index)[index]
970
+ new_index_data = self.with_retries(self.datastore.client.indices.get, index=new_name)[new_name]
971
+ source_aliases = index_data.get("aliases", {})
972
+ reindex_aliases = new_index_data.get("aliases", {})
973
+
974
+ alias_actions = []
975
+ for alias in sorted(set(source_aliases) | set(reindex_aliases) | {self.name}):
976
+ source_alias_data = source_aliases.get(alias, {})
977
+ reindex_alias_data = reindex_aliases.get(alias, {})
978
+
979
+ add_alias_data = {"index": index, "alias": alias}
980
+ add_alias_data.update(source_alias_data)
981
+ if (
982
+ alias == self.name
983
+ or source_alias_data.get("is_write_index", False)
984
+ or reindex_alias_data.get("is_write_index", False)
985
+ ):
986
+ add_alias_data["is_write_index"] = True
987
+
988
+ alias_actions.append({"add": add_alias_data})
989
+
990
+ for alias in reindex_aliases:
991
+ alias_actions.append({"remove": {"index": new_name, "alias": alias}})
992
+
993
+ if alias_actions:
994
+ logger.warning("Restoring aliases to %s", index)
995
+ self.with_retries(self.datastore.client.indices.update_aliases, actions=alias_actions)
996
+
997
+ logger.warning("Deleting leftover reindex index %s", new_name)
998
+ self.with_retries(self.datastore.client.indices.delete, index=new_name)
999
+
1000
+ logger.warning("Unblock write to the index")
1001
+ self.with_retries(
1002
+ self.datastore.client.indices.put_settings,
1003
+ index=index,
1004
+ settings=write_unblock_settings,
1005
+ )
823
1006
 
824
1007
  return True
825
1008
 
@@ -0,0 +1,134 @@
1
+ import argparse
2
+ import json
3
+ import os
4
+ import sys
5
+ import time
6
+
7
+ DELAY = 5
8
+
9
+
10
+ if __name__ == "__main__":
11
+ parser = argparse.ArgumentParser(
12
+ description="Reindex elasticsearch indexes.",
13
+ epilog="Valid index names are derived from the datastore configuration.",
14
+ )
15
+ parser.add_argument("indexes", nargs="*", help="Indexes to reindex.")
16
+ parser.add_argument("--all", action="store_true", help="Reindex all indexes.")
17
+ parser.add_argument("--force", action="store_true", help="Skip confirmation prompts and countdown.")
18
+ parser.add_argument("--verbose", action="store_true", help="Print index schema before reindexing.")
19
+ parser.add_argument(
20
+ "--cleanup",
21
+ action="store_true",
22
+ help="Recover from a failed or interrupted reindex by restoring the source index and "
23
+ "deleting leftover '__reindex' indexes. If the source index is missing, cleanup fails "
24
+ "to avoid deleting the only remaining copy of the data.",
25
+ )
26
+ parser.add_argument(
27
+ "--allow-failures",
28
+ action="store_true",
29
+ help="DESTRUCTIVE: proceed even if the reindex reports document failures, version conflicts, "
30
+ "or a document count mismatch. Documents that cannot be converted to the new mappings will be "
31
+ "permanently dropped. Only use this for intentional lossy migrations.",
32
+ )
33
+ parser.add_argument(
34
+ "--timeout",
35
+ type=int,
36
+ default=3600,
37
+ help="Elasticsearch transport timeout in seconds for synchronous operations. Increase this if "
38
+ "large indexes time out during the clone/settings steps. Default: 3600.",
39
+ )
40
+ args = parser.parse_args()
41
+
42
+ if args.all and args.indexes:
43
+ parser.error("--all cannot be combined with positional index arguments.")
44
+
45
+ if not args.indexes and not args.all:
46
+ parser.error("Provide index names as arguments, or use --all.")
47
+
48
+ if args.timeout <= 0:
49
+ parser.error("--timeout must be a positive number of seconds.")
50
+
51
+ # Raise the Elasticsearch transport timeout before the datastore is imported so the value is
52
+ # baked into every connection (including connections rebuilt after a reset). This lets long
53
+ # running synchronous operations complete without timing out the client.
54
+ os.environ["HWL_DATASTORE_TRANSPORT_TIMEOUT"] = str(args.timeout)
55
+
56
+ from howler.datastore.collection import ESCollection
57
+ from howler.datastore.exceptions import DataStoreException
58
+
59
+ ESCollection.IGNORE_ENSURE_COLLECTION = True
60
+
61
+ if args.force:
62
+ ESCollection.ENSURE_COLLECTION_WARNED = True
63
+
64
+ from howler.common import loader
65
+
66
+ ds = loader.datastore(archive_access=False)
67
+
68
+ # Derive the set of reindexable indexes from the datastore configuration. Collections without
69
+ # an ODM model (e.g. user_avatar) cannot be reindexed and are excluded.
70
+ index_names = sorted(name for name, model in ds.ds.get_models().items() if model is not None)
71
+
72
+ invalid = [name for name in args.indexes if name not in index_names]
73
+ if invalid:
74
+ parser.error(f"Invalid index(es): {', '.join(invalid)}. Valid options: {', '.join(index_names)}")
75
+
76
+ selected = list(dict.fromkeys(index_names if args.all else args.indexes))
77
+
78
+ if args.cleanup:
79
+ for index_name in selected:
80
+ collection: ESCollection = getattr(ds, index_name)
81
+ print(f"Cleaning up leftover reindex state for '{index_name}'.")
82
+ try:
83
+ collection.reindex_cleanup()
84
+ except DataStoreException as e:
85
+ print(f"ERROR: Cleanup of '{index_name}' failed: {e}", file=sys.stderr)
86
+ sys.exit(1)
87
+ print(f"Cleanup of '{index_name}' complete.")
88
+ sys.exit(0)
89
+
90
+ if args.allow_failures and not args.force:
91
+ print(
92
+ "WARNING: --allow-failures is DESTRUCTIVE. Documents that cannot be converted to the new "
93
+ "mappings will be permanently dropped, and count mismatches will be ignored."
94
+ )
95
+ answer = input("Are you sure you want to proceed with --allow-failures? [yes/NO] ")
96
+ if not answer.startswith("y"):
97
+ print("Confirmation not provided, aborting.")
98
+ sys.exit(1)
99
+
100
+ for index_name in selected:
101
+ collection: ESCollection = getattr(ds, index_name)
102
+
103
+ if args.verbose:
104
+ print(f"Index schema for '{index_name}':")
105
+ print(json.dumps(collection._get_index_mappings(), indent=2))
106
+
107
+ print(f"Reindexing: {', '.join(collection.index_list_full)}")
108
+
109
+ if not args.force:
110
+ answer = input(f"Are you sure you want to reindex '{index_name}'? [yes/NO] ")
111
+ if not answer.startswith("y"):
112
+ print("Confirmation not provided, skipping.")
113
+ continue
114
+
115
+ for i in range(2 * DELAY):
116
+ print(f"Reindexing in {2 * DELAY - i}...", end="\r")
117
+ time.sleep(1)
118
+ print()
119
+
120
+ try:
121
+ result = collection.reindex(allow_failures=args.allow_failures, request_timeout=args.timeout)
122
+ except Exception as e: # noqa: BLE001
123
+ print(f"ERROR: Reindex of '{index_name}' failed: {e}", file=sys.stderr)
124
+ print(
125
+ "Run this script with --cleanup to recover before retrying: if the source index is "
126
+ "still present it will be restored and any leftover '__reindex' index removed. If the "
127
+ "source was already deleted but '__reindex' exists, cleanup will fail to avoid deleting "
128
+ "the only remaining copy of the data; recover or delete that index manually. Investigate "
129
+ "the failure before retrying.",
130
+ file=sys.stderr,
131
+ )
132
+ sys.exit(1)
133
+
134
+ print(f"Reindex of '{index_name}' complete. Success: {result}.")
@@ -24,13 +24,16 @@ def assess_hit(
24
24
  assessment: Optional[str] = None,
25
25
  rationale: Optional[str] = None,
26
26
  hit: Optional[Union[dict[str, Any], Hit]] = None,
27
+ *,
28
+ user: User | str,
27
29
  **kwargs,
28
30
  ) -> list[OdmUpdateOperation]:
29
31
  """Update the assessment and esclation of a hit
30
32
 
31
33
  Args:
34
+ user (User | str): The user making the assessment
32
35
  assessment (Optional[str], optional): The assessment to set the hit to. Defaults to None.
33
- hit (Optional[dict[str, Any]], optional): The hit to update. Defaults to None.
36
+ hit (Optional[Union[dict[str, Any], Hit]], optional): The hit to update. Defaults to None.
34
37
 
35
38
  Raises:
36
39
  InvalidDataException: An invalid assessment was provided
@@ -63,10 +66,16 @@ def assess_hit(
63
66
  escalation,
64
67
  )
65
68
 
69
+ if assessment is None:
70
+ assessor_id = None
71
+ else:
72
+ assessor_id = user.get("uname", user.get("username", None)) if isinstance(user, User) else user
73
+
66
74
  return [
67
75
  odm_helper.update("howler.assessment", assessment),
68
76
  odm_helper.update("howler.escalation", escalation),
69
77
  odm_helper.update("howler.rationale", rationale, silent=True),
78
+ odm_helper.update("howler.assessor", assessor_id, silent=True),
70
79
  ]
71
80
 
72
81
 
@@ -168,6 +168,7 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[str], pr
168
168
  hit.howler.status = "open"
169
169
  hit.howler.assignment = "unassigned"
170
170
  hit.howler.escalation = choice([Escalation.HIT, Escalation.ALERT])
171
+ hit.howler.assessor = None
171
172
 
172
173
  if randint(1, 10) > 9:
173
174
  hit.howler.expiry = datetime.now() + timedelta(days=randint(1, 60))
@@ -210,6 +210,12 @@ class HowlerData(odm.Model):
210
210
  description="Unique identifier of the assigned user.",
211
211
  default=DEFAULT_ASSIGNMENT,
212
212
  )
213
+ assessor: Optional[str] = odm.Optional(
214
+ odm.Keyword(
215
+ description="The most recent person to assess a hit",
216
+ default=None,
217
+ )
218
+ )
213
219
  bundles: list[str] = odm.List(
214
220
  odm.Keyword(
215
221
  description="A list of bundle IDs this hit is a part of. Corresponds to the howler.id of the bundle."
@@ -577,6 +577,7 @@ def create_hits(ds: HowlerDatastore, hit_count: int = 200):
577
577
  hit.howler.id,
578
578
  [
579
579
  *assess_hit(
580
+ user=user,
580
581
  assessment=choice(Assessment.list()),
581
582
  rationale=get_random_string(),
582
583
  hit=hit,
@@ -111,7 +111,7 @@ def get_hit_workflow() -> Workflow:
111
111
  "source": [HitStatus.OPEN, HitStatus.IN_PROGRESS],
112
112
  "transition": HitStatusTransition.ASSESS,
113
113
  "dest": HitStatus.RESOLVED,
114
- "actions": [assess_hit, assign_hit],
114
+ "actions": [assess_hit],
115
115
  }
116
116
  ),
117
117
  Transition(
@@ -152,7 +152,7 @@ suppress-none-returning = true
152
152
  [tool.poetry]
153
153
  package-mode = true
154
154
  name = "howler-api"
155
- version = "4.0.0.dev973"
155
+ version = "4.0.0.dev982"
156
156
  description = "Howler - API server"
157
157
  authors = [
158
158
  "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",