howler-api 4.1.0.dev1389__tar.gz → 4.1.0.dev1418__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/PKG-INFO +1 -1
  2. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/transition.py +32 -10
  3. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/action.py +2 -2
  4. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/hit.py +19 -14
  5. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/ingest.py +2 -2
  6. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/collection.py +87 -9
  7. howler_api-4.1.0.dev1418/howler/external/check_duplicates.py +152 -0
  8. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/mixins.py +10 -4
  9. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/case_service.py +11 -10
  10. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/dossier_service.py +2 -2
  11. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/hit_service.py +31 -16
  12. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/compat.py +3 -3
  13. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/pyproject.toml +1 -1
  14. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/README.md +0 -0
  15. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/__init__.py +0 -0
  16. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/__init__.py +0 -0
  17. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_label.py +0 -0
  18. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_to_bundle.py +0 -0
  19. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/add_to_case.py +0 -0
  20. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/change_field.py +0 -0
  21. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/demote.py +0 -0
  22. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/example_plugin.py +0 -0
  23. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/prioritization.py +0 -0
  24. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/promote.py +0 -0
  25. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/remove_from_bundle.py +0 -0
  26. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/actions/remove_label.py +0 -0
  27. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/__init__.py +0 -0
  28. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/base.py +0 -0
  29. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/socket.py +0 -0
  30. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/__init__.py +0 -0
  31. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/analytic.py +0 -0
  32. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/auth.py +0 -0
  33. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/clue.py +0 -0
  34. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/configs.py +0 -0
  35. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/dossier.py +0 -0
  36. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/help.py +0 -0
  37. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/notebook.py +0 -0
  38. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/overview.py +0 -0
  39. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/search.py +0 -0
  40. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/template.py +0 -0
  41. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/tool.py +0 -0
  42. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/user.py +0 -0
  43. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/__init__.py +0 -0
  44. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/etag.py +0 -0
  45. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/utils/params.py +0 -0
  46. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v1/view.py +0 -0
  47. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/__init__.py +0 -0
  48. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/case.py +0 -0
  49. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/fuzzy.py +0 -0
  50. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/api/v2/search.py +0 -0
  51. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/app.py +0 -0
  52. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/README.md +0 -0
  53. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/__init__.py +0 -0
  54. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/classification.py +0 -0
  55. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/classification.yml +0 -0
  56. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/exceptions.py +0 -0
  57. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/loader.py +0 -0
  58. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/__init__.py +0 -0
  59. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/audit.py +0 -0
  60. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/logging/format.py +0 -0
  61. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/net.py +0 -0
  62. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/net_static.py +0 -0
  63. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/random_user.py +0 -0
  64. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/common/swagger.py +0 -0
  65. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/config.py +0 -0
  66. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/__init__.py +0 -0
  67. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/action_queue_worker.py +0 -0
  68. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/correlation.py +0 -0
  69. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/retention.py +0 -0
  70. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/cronjobs/view_cleanup.py +0 -0
  71. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/README.md +0 -0
  72. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/__init__.py +0 -0
  73. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/bulk.py +0 -0
  74. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/constants.py +0 -0
  75. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/exceptions.py +0 -0
  76. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/howler_store.py +0 -0
  77. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/migrations/fix_process.py +0 -0
  78. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/operations.py +0 -0
  79. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/schemas.py +0 -0
  80. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/store.py +0 -0
  81. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/__init__.py +0 -0
  82. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/build.py +0 -0
  83. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/support/schemas.py +0 -0
  84. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/types.py +0 -0
  85. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/datastore/utils.py +0 -0
  86. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/error.py +0 -0
  87. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/README.md +0 -0
  88. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/__init__.py +0 -0
  89. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_mitre.py +0 -0
  90. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_sigma_rules.py +0 -0
  91. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/generate_tlds.py +0 -0
  92. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/reindex_data.py +0 -0
  93. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/external/wipe_databases.py +0 -0
  94. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/gunicorn_config.py +0 -0
  95. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/healthz.py +0 -0
  96. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/__init__.py +0 -0
  97. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/azure.py +0 -0
  98. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/discover.py +0 -0
  99. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/hit.py +0 -0
  100. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/oauth.py +0 -0
  101. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/search.py +0 -0
  102. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/workflow.py +0 -0
  103. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/helper/ws.py +0 -0
  104. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/README.md +0 -0
  105. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/__init__.py +0 -0
  106. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/base.py +0 -0
  107. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/charter.txt +0 -0
  108. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/constants.py +0 -0
  109. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/helper.py +0 -0
  110. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/howler_enum.py +0 -0
  111. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/__init__.py +0 -0
  112. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/action.py +0 -0
  113. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/analytic.py +0 -0
  114. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/assemblyline.py +0 -0
  115. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/aws.py +0 -0
  116. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/azure.py +0 -0
  117. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/case.py +0 -0
  118. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/cbs.py +0 -0
  119. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/clue.py +0 -0
  120. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/config.py +0 -0
  121. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/dossier.py +0 -0
  122. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/__init__.py +0 -0
  123. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/agent.py +0 -0
  124. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/autonomous_system.py +0 -0
  125. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/client.py +0 -0
  126. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/cloud.py +0 -0
  127. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/code_signature.py +0 -0
  128. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/container.py +0 -0
  129. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/dns.py +0 -0
  130. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/egress.py +0 -0
  131. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/elf.py +0 -0
  132. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/email.py +0 -0
  133. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/error.py +0 -0
  134. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/event.py +0 -0
  135. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/faas.py +0 -0
  136. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/file.py +0 -0
  137. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/geo.py +0 -0
  138. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/group.py +0 -0
  139. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/hash.py +0 -0
  140. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/host.py +0 -0
  141. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/http.py +0 -0
  142. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/ingress.py +0 -0
  143. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/interface.py +0 -0
  144. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/network.py +0 -0
  145. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/observer.py +0 -0
  146. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/organization.py +0 -0
  147. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/os.py +0 -0
  148. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/pe.py +0 -0
  149. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/process.py +0 -0
  150. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/registry.py +0 -0
  151. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/related.py +0 -0
  152. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/rule.py +0 -0
  153. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/server.py +0 -0
  154. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/threat.py +0 -0
  155. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/tls.py +0 -0
  156. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/url.py +0 -0
  157. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/user.py +0 -0
  158. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/user_agent.py +0 -0
  159. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/ecs/vulnerability.py +0 -0
  160. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/event.py +0 -0
  161. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/gcp.py +0 -0
  162. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/hit.py +0 -0
  163. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/howler_data.py +0 -0
  164. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/lead.py +0 -0
  165. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/localized_label.py +0 -0
  166. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/overview.py +0 -0
  167. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/pivot.py +0 -0
  168. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/record.py +0 -0
  169. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/template.py +0 -0
  170. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/user.py +0 -0
  171. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/models/view.py +0 -0
  172. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/random_data.py +0 -0
  173. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/odm/randomizer.py +0 -0
  174. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/patched.py +0 -0
  175. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/plugins/__init__.py +0 -0
  176. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/plugins/config.py +0 -0
  177. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/__init__.py +0 -0
  178. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/README.md +0 -0
  179. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/__init__.py +0 -0
  180. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/counters.py +0 -0
  181. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/events.py +0 -0
  182. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/hash.py +0 -0
  183. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/lock.py +0 -0
  184. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/__init__.py +0 -0
  185. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/comms.py +0 -0
  186. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/multi.py +0 -0
  187. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/named.py +0 -0
  188. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/queues/priority.py +0 -0
  189. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/set.py +0 -0
  190. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/remote/datatypes/user_quota_tracker.py +0 -0
  191. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/__init__.py +0 -0
  192. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/socket.py +0 -0
  193. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/security/utils.py +0 -0
  194. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/__init__.py +0 -0
  195. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/action_service.py +0 -0
  196. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/analytic_service.py +0 -0
  197. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/auth_service.py +0 -0
  198. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/bundle_compat_service.py +0 -0
  199. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/comms_service.py +0 -0
  200. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/config_service.py +0 -0
  201. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/correlation_service.py +0 -0
  202. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/docs_service.py +0 -0
  203. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/event_service.py +0 -0
  204. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/fuzzy_service.py +0 -0
  205. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/jwt_service.py +0 -0
  206. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/lucene_service.py +0 -0
  207. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/notebook_service.py +0 -0
  208. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/overview_service.py +0 -0
  209. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/search_service.py +0 -0
  210. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/template_service.py +0 -0
  211. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/user_service.py +0 -0
  212. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/services/viewer_service.py +0 -0
  213. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/telemetry.py +0 -0
  214. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/__init__.py +0 -0
  215. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/annotations.py +0 -0
  216. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/chunk.py +0 -0
  217. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/constants.py +0 -0
  218. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/dict_utils.py +0 -0
  219. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/indexes.py +0 -0
  220. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/isotime.py +0 -0
  221. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/list_utils.py +0 -0
  222. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/lucene.py +0 -0
  223. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/net_utils.py +0 -0
  224. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/path.py +0 -0
  225. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/socket_utils.py +0 -0
  226. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/str_utils.py +0 -0
  227. {howler_api-4.1.0.dev1389 → howler_api-4.1.0.dev1418}/howler/utils/uid.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: howler-api
3
- Version: 4.1.0.dev1389
3
+ Version: 4.1.0.dev1418
4
4
  Summary: Howler - API server
5
5
  License: MIT
6
6
  Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
@@ -5,6 +5,7 @@ from howler.actions import check_hit_limit
5
5
  from howler.common.exceptions import InvalidDataException, NotFoundException
6
6
  from howler.common.loader import datastore
7
7
  from howler.common.logging import get_logger
8
+ from howler.datastore.exceptions import VersionConflictException
8
9
  from howler.helper.workflow import Workflow, WorkflowException
9
10
  from howler.odm.models.action import VALID_TRIGGERS
10
11
  from howler.odm.models.howler_data import (
@@ -20,11 +21,39 @@ from howler.utils.list_utils import flatten_list
20
21
  OPERATION_ID = "transition"
21
22
  MAX_HITS_BASIC = 10
22
23
  MAX_HITS_ADVANCED = 1000
24
+ MAX_VERSION_CONFLICT_ATTEMPTS = 3
23
25
  SKIP_CENTRAL_LIMIT = True # This operation transforms the query, handles limit check locally
24
26
 
25
27
  log = get_logger(__file__)
26
28
 
27
29
 
30
+ def _transition_failure_report(hit_id: str, error: Exception) -> dict[str, str]:
31
+ if isinstance(error, VersionConflictException):
32
+ return {
33
+ "query": f"howler.id:{hit_id}",
34
+ "outcome": "error",
35
+ "title": "Version Conflict",
36
+ "message": "The hit was modified while this transition was running.",
37
+ }
38
+
39
+ return {
40
+ "query": f"howler.id:{hit_id}",
41
+ "outcome": "error",
42
+ "title": "An error occurred while processing.",
43
+ "message": str(error),
44
+ }
45
+
46
+
47
+ def _transition_hit_with_retry(hit_id: str, transition: HitStatusTransition, user: User, **kwargs) -> None:
48
+ for attempt in range(MAX_VERSION_CONFLICT_ATTEMPTS):
49
+ try:
50
+ hit_service.transition_hit(hit_id, transition, user, **kwargs)
51
+ return
52
+ except VersionConflictException:
53
+ if attempt == MAX_VERSION_CONFLICT_ATTEMPTS - 1:
54
+ raise
55
+
56
+
28
57
  def __parse_workflow_actions(workflow: Workflow) -> dict[str, set[str]]:
29
58
  """Take in a workflow, and parse the steps and transitions of that workflow into a format understood by the UI"""
30
59
  parsed_args: dict[str, set[str]] = {}
@@ -126,22 +155,15 @@ def execute(
126
155
  total_processed = 0
127
156
  for hit_id in ids:
128
157
  try:
129
- hit_service.transition_hit(
158
+ _transition_hit_with_retry(
130
159
  hit_id,
131
160
  cast(HitStatusTransition, HitStatusTransition[transition]),
132
161
  user,
133
162
  **kwargs,
134
163
  )
135
164
  success_ids.add(hit_id)
136
- except (InvalidDataException, NotFoundException, WorkflowException) as e:
137
- report.append(
138
- {
139
- "query": f"howler.id:{hit_id}",
140
- "outcome": "error",
141
- "title": "An error occurred while processing.",
142
- "message": str(e),
143
- }
144
- )
165
+ except (InvalidDataException, NotFoundException, VersionConflictException, WorkflowException) as e:
166
+ report.append(_transition_failure_report(hit_id, e))
145
167
 
146
168
  total_processed += 1
147
169
  if total_processed % 10 == 0:
@@ -143,7 +143,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
143
143
 
144
144
  ds = datastore()
145
145
 
146
- existing_action = ds.action.get(id, as_obj=False)
146
+ existing_action, server_version = ds.action.get(id, as_obj=False, version=True)
147
147
 
148
148
  if not existing_action:
149
149
  return not_found(err="The specified automation does not exist")
@@ -166,7 +166,7 @@ def update_action(id: str, user: User, **kwargs) -> Response:
166
166
  action_obj = Action(updated_action)
167
167
  action_obj.action_id = id
168
168
 
169
- ds.action.save(action_obj.action_id, action_obj, refresh=refresh)
169
+ ds.action.save(action_obj.action_id, action_obj, version=server_version, refresh=refresh)
170
170
  except HowlerException as e:
171
171
  return bad_request(err=str(e))
172
172
 
@@ -528,7 +528,7 @@ def get_assigned_hits(user, **kwargs):
528
528
  @api_login(audit=False, required_priv=["W"])
529
529
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
530
530
  @parse_parameters(refresh=parse_refresh)
531
- def add_label(id, label_set, user, **kwargs):
531
+ def add_label(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
532
532
  """Add labels to a hit.
533
533
 
534
534
  Variables:
@@ -575,7 +575,8 @@ def add_label(id, label_set, user, **kwargs):
575
575
  hit_service.update_hit(
576
576
  id,
577
577
  [hit_helper.list_add(f"howler.labels.{label_set}", label) for label in labels],
578
- user["uname"],
578
+ user.uname,
579
+ version=server_version,
579
580
  refresh="true" if DEBUG_FORCE_REFRESH else refresh,
580
581
  )
581
582
 
@@ -591,7 +592,7 @@ def add_label(id, label_set, user, **kwargs):
591
592
  @api_login(audit=False, required_priv=["W"])
592
593
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
593
594
  @parse_parameters(refresh=parse_refresh)
594
- def remove_labels(id, label_set, user, **kwargs):
595
+ def remove_labels(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
595
596
  """Remove labels from a hit.
596
597
 
597
598
  Variables:
@@ -632,7 +633,8 @@ def remove_labels(id, label_set, user, **kwargs):
632
633
  hit_service.update_hit(
633
634
  id,
634
635
  [hit_helper.list_remove(f"howler.labels.{label_set}", label) for label in labels],
635
- user["uname"],
636
+ user.uname,
637
+ version=server_version,
636
638
  refresh="true" if DEBUG_FORCE_REFRESH else refresh,
637
639
  )
638
640
 
@@ -648,7 +650,7 @@ def remove_labels(id, label_set, user, **kwargs):
648
650
  @api_login(audit=False, required_priv=["W"])
649
651
  @add_etag(getter=hit_service.get_hit, check_if_match=True)
650
652
  @parse_parameters(refresh=parse_refresh)
651
- def transition(id: str, user: User, **kwargs):
653
+ def transition(id: str, user: User, server_version: str | None = None, **kwargs):
652
654
  """Transition a hit
653
655
 
654
656
  Variables:
@@ -679,11 +681,12 @@ def transition(id: str, user: User, **kwargs):
679
681
  return bad_request(err="Invalid data format")
680
682
 
681
683
  transition = transition_data["transition"]
684
+ version: str | None = None
682
685
  if "If-Match" in request.headers:
683
686
  version = request.headers["If-Match"]
684
687
  else:
685
688
  logger.warning("User is mising version - no If-Match header in request.")
686
- version = None
689
+ version = server_version
687
690
 
688
691
  try:
689
692
  if transition not in HitStatusTransition.list():
@@ -694,7 +697,7 @@ def transition(id: str, user: User, **kwargs):
694
697
  )
695
698
  )
696
699
 
697
- hit_service.transition_hit(
700
+ hit, version = hit_service.transition_hit(
698
701
  id, transition, user, version, refresh=refresh, **kwargs, **transition_data.get("data", {})
699
702
  )
700
703
  except (WorkflowException, DataStoreException, InvalidDataException) as e:
@@ -704,7 +707,6 @@ def transition(id: str, user: User, **kwargs):
704
707
  except HowlerException as e:
705
708
  return internal_error(err=str(e))
706
709
 
707
- hit, version = hit_service.get_hit(id, as_odm=False, version=True)
708
710
  return ok(hit), version
709
711
 
710
712
 
@@ -712,7 +714,7 @@ def transition(id: str, user: User, **kwargs):
712
714
  @hit_api.route("/<id>/comments/<comment_id>", methods=["GET"])
713
715
  @api_login(audit=False, required_priv=["R"])
714
716
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
715
- def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwargs):
717
+ def get_comment(id: str, comment_id: str, user: User, server_version: str | None = None, **kwargs):
716
718
  """Get a comment associated with a particular hit
717
719
 
718
720
  Variables:
@@ -741,7 +743,7 @@ def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwa
741
743
  @hit_api.route("/<id>/comments", methods=["POST"])
742
744
  @api_login(audit=False, required_priv=["W"])
743
745
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
744
- def add_comment(id: str, user: dict[str, Any], **kwargs):
746
+ def add_comment(id: str, user: User, server_version: str | None = None, **kwargs):
745
747
  """Add a comment
746
748
 
747
749
  Variables:
@@ -781,12 +783,13 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
781
783
  [
782
784
  hit_helper.list_add(
783
785
  "howler.comment",
784
- Comment({"user": user["uname"], "value": comment_value}),
786
+ Comment({"user": user.uname, "value": comment_value}),
785
787
  explanation=f"Added a comment:\n\n{comment_value}",
786
788
  if_missing=True,
787
789
  ),
788
790
  ],
789
- user["uname"],
791
+ user.uname,
792
+ version=server_version,
790
793
  )
791
794
  except DataStoreException as e:
792
795
  return bad_request(err=str(e))
@@ -800,7 +803,7 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
800
803
  @hit_api.route("/<id>/comments/<comment_id>", methods=["PUT"])
801
804
  @api_login(audit=False, required_priv=["W"])
802
805
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
803
- def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
806
+ def edit_comment(id: str, comment_id: str, user: dict[str, Any], server_version: str | None = None, **kwargs):
804
807
  """Edit a comment
805
808
 
806
809
  Variables:
@@ -865,6 +868,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
865
868
  ),
866
869
  ],
867
870
  user["uname"],
871
+ version=server_version,
868
872
  )
869
873
 
870
874
  return ok(new_hit), version
@@ -874,7 +878,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
874
878
  @hit_api.route("/<id>/comments", methods=["DELETE"])
875
879
  @api_login(audit=False, required_priv=["W"])
876
880
  @add_etag(getter=hit_service.get_hit, check_if_match=False)
877
- def delete_comments(id: str, user: User, **kwargs):
881
+ def delete_comments(id: str, user: User, server_version: str | None = None, **kwargs):
878
882
  """Delete a set of comments
879
883
 
880
884
  Variables:
@@ -923,6 +927,7 @@ def delete_comments(id: str, user: User, **kwargs):
923
927
  for comment in comments
924
928
  ],
925
929
  user.uname,
930
+ version=server_version,
926
931
  )
927
932
 
928
933
  except DataStoreException as e:
@@ -265,7 +265,7 @@ def overwrite(index: str, id: str, **kwargs):
265
265
 
266
266
  ds = datastore()
267
267
 
268
- record = ds[index].get(id, as_obj=False, version=False)
268
+ record, server_version = ds[index].get(id, as_obj=False, version=True)
269
269
  if not record:
270
270
  return not_found(err="Record %s does not exist" % id)
271
271
 
@@ -292,7 +292,7 @@ def overwrite(index: str, id: str, **kwargs):
292
292
  ds[index].save(
293
293
  id,
294
294
  odm(new_record) if odm else new_record,
295
- version=kwargs.get("server_version"),
295
+ version=server_version,
296
296
  refresh=refresh,
297
297
  )
298
298
 
@@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, Generic, Literal, Optional, TypeVar, Uni
16
16
  import elasticsearch
17
17
  from datemath import dm
18
18
  from datemath.helpers import DateMathException
19
+ from elastic_transport import ApiResponseMeta
19
20
  from opentelemetry import trace
20
21
 
21
22
  from howler import odm
@@ -1136,13 +1137,46 @@ class ESCollection(Generic[ModelType]):
1136
1137
 
1137
1138
  return data
1138
1139
 
1140
+ def _search_exists(self, key) -> bool:
1141
+ """Check document existence with an alias-safe search query."""
1142
+ result = self.with_retries(
1143
+ self.datastore.client.search,
1144
+ index=self.name,
1145
+ query={"ids": {"values": [key]}},
1146
+ size=0,
1147
+ track_total_hits=True,
1148
+ )
1149
+ total = result["hits"]["total"]
1150
+ return total["value"] > 0 if isinstance(total, dict) else total > 0
1151
+
1139
1152
  def exists(self, key) -> bool:
1140
1153
  """Check if a document exists in the datastore.
1141
1154
 
1142
1155
  :param key: key of the document to get from the datastore
1143
1156
  :return: true/false depending if the document exists or not
1144
1157
  """
1145
- return cast(bool, self.with_retries(self.datastore.client.exists, index=self.name, id=key, _source=False))
1158
+ if self.ilm_config:
1159
+ return self._search_exists(key)
1160
+
1161
+ try:
1162
+ return cast(bool, self.with_retries(self.datastore.client.exists, index=self.name, id=key, _source=False))
1163
+ except elasticsearch.BadRequestError:
1164
+ logger.exception(
1165
+ "Single-document existence check failed on %s; falling back to an alias-safe search.",
1166
+ self.name,
1167
+ )
1168
+ return self._search_exists(key)
1169
+
1170
+ def _raise_document_not_found(self, key: str, result: Any) -> typing.NoReturn:
1171
+ """Raise the same error shape as an Elasticsearch document lookup for an empty search."""
1172
+ meta = ApiResponseMeta(
1173
+ status=404,
1174
+ http_version="1.1",
1175
+ headers=cast(Any, {}),
1176
+ duration=0.0,
1177
+ node=cast(Any, None),
1178
+ )
1179
+ raise elasticsearch.exceptions.NotFoundError(f"Document with id {key} not found", meta, result)
1146
1180
 
1147
1181
  @overload
1148
1182
  def _get(self, key, retries, version: Literal[False]) -> dict[str, Any]: ...
@@ -1154,7 +1188,7 @@ class ESCollection(Generic[ModelType]):
1154
1188
  """Versioned get-save for atomic update has two paths:
1155
1189
  1. Document doesn't exist at all. Create token will be returned for version.
1156
1190
  This way only the first query to try and create the document will succeed.
1157
- 2. Document exists in hot. A version string with the info needed to do a versioned save is returned.
1191
+ 2. Document exists. A version string with the info needed to do a versioned save is returned.
1158
1192
 
1159
1193
  The create token is needed to differentiate between "I'm saving a new
1160
1194
  document non-atomic (version=None)" and "I'm saving a new document
@@ -1173,11 +1207,29 @@ class ESCollection(Generic[ModelType]):
1173
1207
  done = False
1174
1208
  while not done:
1175
1209
  try:
1176
- doc = self.with_retries(self.datastore.client.get, index=self.name, id=key)
1210
+ if self.ilm_config:
1211
+ result = self.with_retries(
1212
+ self.datastore.client.search,
1213
+ index=self.name,
1214
+ query={"ids": {"values": [key]}},
1215
+ size=1,
1216
+ seq_no_primary_term=True,
1217
+ )
1218
+ hits = result["hits"]["hits"]
1219
+ if not hits:
1220
+ self._raise_document_not_found(key, result)
1221
+ doc = hits[0]
1222
+ else:
1223
+ doc = self.with_retries(self.datastore.client.get, index=self.name, id=key)
1224
+
1177
1225
  if version:
1226
+ if self.ilm_config:
1227
+ version_token = f"{doc['_index']}---{doc['_seq_no']}---{doc['_primary_term']}"
1228
+ else:
1229
+ version_token = f"{doc['_seq_no']}---{doc['_primary_term']}"
1178
1230
  return (
1179
1231
  normalize_output(doc["_source"]),
1180
- f"{doc['_seq_no']}---{doc['_primary_term']}",
1232
+ version_token,
1181
1233
  )
1182
1234
  return normalize_output(doc["_source"])
1183
1235
  except elasticsearch.exceptions.NotFoundError:
@@ -1196,6 +1248,19 @@ class ESCollection(Generic[ModelType]):
1196
1248
 
1197
1249
  return None
1198
1250
 
1251
+ def _get_version_write_target(self, version: str) -> tuple[str, str, str]:
1252
+ """Return the concrete write target and optimistic-concurrency values from a version token."""
1253
+ version_parts = version.split("---")
1254
+ if len(version_parts) == 3:
1255
+ index, seq_no, primary_term = version_parts
1256
+ return index, seq_no, primary_term
1257
+
1258
+ if len(version_parts) == 2:
1259
+ seq_no, primary_term = version_parts
1260
+ return self.name, seq_no, primary_term
1261
+
1262
+ raise DataStoreException(f"Invalid version token for {self.name}: {version!r}")
1263
+
1199
1264
  @overload
1200
1265
  def get(self, key, as_obj: Literal[True], version: Literal[True]) -> tuple[ModelType | None, str]: ...
1201
1266
 
@@ -1325,13 +1390,17 @@ class ESCollection(Generic[ModelType]):
1325
1390
 
1326
1391
  saved_data["id"] = key
1327
1392
  operation = "index"
1393
+ index = self.name
1328
1394
  seq_no = None
1329
1395
  primary_term = None
1330
1396
 
1397
+ if version is None and self.ilm_config:
1398
+ _, version = self.get_if_exists(key, as_obj=False, version=True)
1399
+
1331
1400
  if version == CREATE_TOKEN:
1332
1401
  operation = "create"
1333
1402
  elif version:
1334
- seq_no, primary_term = version.split("---")
1403
+ index, seq_no, primary_term = self._get_version_write_target(version)
1335
1404
 
1336
1405
  if refresh == "true":
1337
1406
  logger.warning(
@@ -1341,7 +1410,7 @@ class ESCollection(Generic[ModelType]):
1341
1410
  try:
1342
1411
  self.with_retries(
1343
1412
  self.datastore.client.index,
1344
- index=self.name,
1413
+ index=index,
1345
1414
  id=key,
1346
1415
  document=json.dumps(saved_data),
1347
1416
  op_type=operation,
@@ -1541,15 +1610,20 @@ class ESCollection(Generic[ModelType]):
1541
1610
  """
1542
1611
  operations = self._validate_operations(operations)
1543
1612
  script = self._create_scripts_from_operations(operations)
1613
+ index = self.name
1544
1614
  seq_no = None
1545
1615
  primary_term = None
1616
+
1617
+ if version is None and self.ilm_config:
1618
+ _, version = self.get_if_exists(key, as_obj=False, version=True)
1619
+
1546
1620
  if version:
1547
- seq_no, primary_term = version.split("---")
1621
+ index, seq_no, primary_term = self._get_version_write_target(version)
1548
1622
 
1549
1623
  try:
1550
1624
  res = self.with_retries(
1551
1625
  self.datastore.client.update,
1552
- index=self.name,
1626
+ index=index,
1553
1627
  id=key,
1554
1628
  script=script,
1555
1629
  if_seq_no=seq_no,
@@ -1559,7 +1633,11 @@ class ESCollection(Generic[ModelType]):
1559
1633
  )
1560
1634
  return (
1561
1635
  res["result"] == "updated",
1562
- f"{res['_seq_no']}---{res['_primary_term']}",
1636
+ (
1637
+ f"{res['_index']}---{res['_seq_no']}---{res['_primary_term']}"
1638
+ if self.ilm_config
1639
+ else f"{res['_seq_no']}---{res['_primary_term']}"
1640
+ ),
1563
1641
  )
1564
1642
  except elasticsearch.NotFoundError as e:
1565
1643
  logger.warning("Update - elasticsearch.NotFoundError: %s %s", e.message, e.info)
@@ -0,0 +1,152 @@
1
+ """Find and optionally remove duplicate documents from an ILM collection."""
2
+
3
+ import argparse
4
+ import heapq
5
+ import sys
6
+ from collections.abc import Iterator
7
+ from typing import Any
8
+
9
+
10
+ def iter_index_ids(collection: Any, index: str, batch_size: int) -> Iterator[str]:
11
+ """Yield document IDs from *index* in sorted order without retaining prior pages."""
12
+ search_after = None
13
+
14
+ while True:
15
+ search_args = {
16
+ "index": index,
17
+ "query": {"match_all": {}},
18
+ "sort": [{"id": "asc"}],
19
+ "size": batch_size,
20
+ "_source": False,
21
+ "track_total_hits": False,
22
+ }
23
+ if search_after is not None:
24
+ search_args["search_after"] = search_after
25
+
26
+ response = collection.with_retries(collection.datastore.client.search, **search_args)
27
+ hits = response["hits"]["hits"]
28
+ if not hits:
29
+ return
30
+
31
+ for hit in hits:
32
+ yield hit["_id"]
33
+
34
+ search_after = hits[-1]["sort"]
35
+
36
+
37
+ def _push_next_id(heap: list[tuple[str, str, Iterator[str]]], index: str, document_ids: Iterator[str]) -> None:
38
+ try:
39
+ heapq.heappush(heap, (next(document_ids), index, document_ids))
40
+ except StopIteration:
41
+ pass
42
+
43
+
44
+ def iter_duplicates(collection: Any, indexes: list[str], batch_size: int) -> Iterator[tuple[str, list[str]]]:
45
+ """Yield each duplicated ID and the physical indexes that contain it.
46
+
47
+ This is a k-way merge over paginated, sorted searches. Its memory use is bounded by
48
+ the number of physical indexes and one Elasticsearch result page per active iterator.
49
+ """
50
+ heap: list[tuple[str, str, Iterator[str]]] = []
51
+ for index in indexes:
52
+ document_ids = iter_index_ids(collection, index, batch_size)
53
+ _push_next_id(heap, index, document_ids)
54
+
55
+ while heap:
56
+ document_id, index, document_ids = heapq.heappop(heap)
57
+ duplicate_indexes = [index]
58
+ _push_next_id(heap, index, document_ids)
59
+
60
+ while heap and heap[0][0] == document_id:
61
+ _, index, document_ids = heapq.heappop(heap)
62
+ duplicate_indexes.append(index)
63
+ _push_next_id(heap, index, document_ids)
64
+
65
+ if len(duplicate_indexes) > 1:
66
+ yield document_id, duplicate_indexes
67
+
68
+
69
+ def delete_duplicates(collection: Any, document_id: str, indexes: list[str]) -> int:
70
+ """Delete all supplied physical-index copies of a document ID."""
71
+ deleted = 0
72
+ for index in indexes:
73
+ response = collection.with_retries(collection.datastore.client.delete, index=index, id=document_id)
74
+ if response["result"] == "deleted":
75
+ deleted += 1
76
+ return deleted
77
+
78
+
79
+ def main() -> int:
80
+ """Run the interactive duplicate scan for an ILM collection."""
81
+ parser = argparse.ArgumentParser(
82
+ description="Find duplicate document IDs across an ILM collection's physical indexes."
83
+ )
84
+ parser.add_argument("collection", help="Collection name registered in the datastore.")
85
+ parser.add_argument(
86
+ "--batch-size",
87
+ type=int,
88
+ default=1_000,
89
+ help="IDs fetched from each physical index per Elasticsearch request (default: 1000).",
90
+ )
91
+ args = parser.parse_args()
92
+
93
+ if args.batch_size <= 0:
94
+ parser.error("--batch-size must be a positive integer.")
95
+
96
+ from howler.datastore.collection import ESCollection
97
+
98
+ # Checking duplicates must not create collections or alter their ILM configuration.
99
+ ESCollection.IGNORE_ENSURE_COLLECTION = True
100
+ ESCollection.ENSURE_COLLECTION_WARNED = True
101
+
102
+ from howler.common import loader
103
+
104
+ datastore = loader.datastore(archive_access=False)
105
+ collection_names = datastore.ds.get_models()
106
+ if args.collection not in collection_names:
107
+ parser.error(
108
+ f"Unknown collection '{args.collection}'. Valid collections: {', '.join(sorted(collection_names))}"
109
+ )
110
+
111
+ collection = getattr(datastore, args.collection)
112
+ if collection.ilm_config is None:
113
+ print(f"Collection '{args.collection}' is not ILM-enabled; nothing to check.")
114
+ return 0
115
+
116
+ collection._refresh_ilm_index_name()
117
+ indexes = collection.index_list_full
118
+ if len(indexes) < 2:
119
+ print(f"Collection '{args.collection}' has fewer than two physical ILM indexes; nothing to check.")
120
+ return 0
121
+
122
+ print(f"Scanning for duplicate IDs in: {', '.join(indexes)}")
123
+ duplicates_found = 0
124
+ copies_deleted = 0
125
+ index_order = {index: position for position, index in enumerate(indexes)}
126
+
127
+ try:
128
+ for document_id, duplicate_indexes in iter_duplicates(collection, indexes, args.batch_size):
129
+ duplicates_found += 1
130
+ keep_index = min(duplicate_indexes, key=index_order.__getitem__)
131
+ stale_indexes = [index for index in duplicate_indexes if index != keep_index]
132
+ print(
133
+ f"Duplicate ID '{document_id}' found in {', '.join(duplicate_indexes)}. "
134
+ f"Keeping the newest copy in '{keep_index}'."
135
+ )
136
+ answer = input(f"Delete copies from {', '.join(stale_indexes)}? [yes/NO] ")
137
+ if answer.lower() in {"y", "yes"}:
138
+ deleted_count = delete_duplicates(collection, document_id, stale_indexes)
139
+ copies_deleted += deleted_count
140
+ print(f"Deleted {deleted_count} copy/copies of '{document_id}'.")
141
+ except KeyboardInterrupt:
142
+ print("\nInterrupted; all completed deletions have been retained.", file=sys.stderr)
143
+ return 130
144
+ finally:
145
+ datastore.ds.close()
146
+
147
+ print(f"Scan complete. Found {duplicates_found} duplicate ID(s); deleted {copies_deleted} stale copy/copies.")
148
+ return 0
149
+
150
+
151
+ if __name__ == "__main__":
152
+ sys.exit(main())
@@ -15,6 +15,7 @@ from howler.common.exceptions import HowlerRuntimeError
15
15
  from howler.common.loader import datastore
16
16
  from howler.datastore.collection import ESCollection
17
17
  from howler.odm.base import Model
18
+ from howler.utils.compat import Self
18
19
 
19
20
  ModelType = TypeVar("ModelType", bound=Model)
20
21
 
@@ -28,10 +29,10 @@ class _ObjectsDescriptor(Generic[ModelType]):
28
29
  """
29
30
 
30
31
  @overload
31
- def __get__(self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
32
+ def __get__(self: Self, obj: None, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
32
33
 
33
34
  @overload
34
- def __get__(self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
35
+ def __get__(self: Self, obj: ModelType, objtype: type[ModelType]) -> ESCollection[ModelType]: ...
35
36
 
36
37
  def __get__(self, obj: ModelType | None, objtype: type[ModelType] | None = None) -> ESCollection[ModelType]:
37
38
  """Return the ESCollection for the owner class.
@@ -82,7 +83,11 @@ class DatastoreMixin(Generic[ModelType]):
82
83
  """
83
84
  return datastore()
84
85
 
85
- def save(self, refresh: Literal["true", "false", "wait_for"] | None = None) -> bool:
86
+ def save(
87
+ self: Self,
88
+ refresh: Literal["true", "false", "wait_for"] | None = None,
89
+ version: str | None = None,
90
+ ) -> bool:
86
91
  """Persist the current model instance to the datastore.
87
92
 
88
93
  Determines the target index from the lowercase class name, extracts the
@@ -96,5 +101,6 @@ class DatastoreMixin(Generic[ModelType]):
96
101
  index_name = self.__class__.__name__.lower()
97
102
  id_field = self.__class__._Model__id_field # type: ignore[attr-defined]
98
103
  current_id = attrgetter(id_field)(self)
104
+ collection = self.ds[index_name]
99
105
 
100
- return self.ds[index_name].save(current_id, self, refresh=refresh)
106
+ return collection.save(current_id, self, version=version, refresh=refresh)