howler-api 4.1.0.dev1405__tar.gz → 4.1.0.dev1430__tar.gz

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.1.0.dev1405
+Version: 4.1.0.dev1430
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/actions/transition.py

@@ -5,6 +5,7 @@ from howler.actions import check_hit_limit
 from howler.common.exceptions import InvalidDataException, NotFoundException
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
+from howler.datastore.exceptions import VersionConflictException
 from howler.helper.workflow import Workflow, WorkflowException
 from howler.odm.models.action import VALID_TRIGGERS
 from howler.odm.models.howler_data import (
@@ -20,11 +21,39 @@ from howler.utils.list_utils import flatten_list
 OPERATION_ID = "transition"
 MAX_HITS_BASIC = 10
 MAX_HITS_ADVANCED = 1000
+MAX_VERSION_CONFLICT_ATTEMPTS = 3
 SKIP_CENTRAL_LIMIT = True  # This operation transforms the query, handles limit check locally
 
 log = get_logger(__file__)
 
 
+def _transition_failure_report(hit_id: str, error: Exception) -> dict[str, str]:
+    if isinstance(error, VersionConflictException):
+        return {
+            "query": f"howler.id:{hit_id}",
+            "outcome": "error",
+            "title": "Version Conflict",
+            "message": "The hit was modified while this transition was running.",
+        }
+
+    return {
+        "query": f"howler.id:{hit_id}",
+        "outcome": "error",
+        "title": "An error occurred while processing.",
+        "message": str(error),
+    }
+
+
+def _transition_hit_with_retry(hit_id: str, transition: HitStatusTransition, user: User, **kwargs) -> None:
+    for attempt in range(MAX_VERSION_CONFLICT_ATTEMPTS):
+        try:
+            hit_service.transition_hit(hit_id, transition, user, **kwargs)
+            return
+        except VersionConflictException:
+            if attempt == MAX_VERSION_CONFLICT_ATTEMPTS - 1:
+                raise
+
+
 def __parse_workflow_actions(workflow: Workflow) -> dict[str, set[str]]:
     """Take in a workflow, and parse the steps and transitions of that workflow into a format understood by the UI"""
     parsed_args: dict[str, set[str]] = {}
@@ -126,22 +155,15 @@ def execute(
     total_processed = 0
     for hit_id in ids:
         try:
-            hit_service.transition_hit(
+            _transition_hit_with_retry(
                 hit_id,
                 cast(HitStatusTransition, HitStatusTransition[transition]),
                 user,
                 **kwargs,
             )
             success_ids.add(hit_id)
-        except (InvalidDataException, NotFoundException, WorkflowException) as e:
-            report.append(
-                {
-                    "query": f"howler.id:{hit_id}",
-                    "outcome": "error",
-                    "title": "An error occurred while processing.",
-                    "message": str(e),
-                }
-            )
+        except (InvalidDataException, NotFoundException, VersionConflictException, WorkflowException) as e:
+            report.append(_transition_failure_report(hit_id, e))
 
         total_processed += 1
         if total_processed % 10 == 0:

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/api/v1/hit.py

@@ -528,7 +528,7 @@ def get_assigned_hits(user, **kwargs):
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
 @parse_parameters(refresh=parse_refresh)
-def add_label(id, label_set, user, **kwargs):
+def add_label(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
     """Add labels to a hit.
 
     Variables:
@@ -575,7 +575,8 @@ def add_label(id, label_set, user, **kwargs):
     hit_service.update_hit(
         id,
         [hit_helper.list_add(f"howler.labels.{label_set}", label) for label in labels],
-        user["uname"],
+        user.uname,
+        version=server_version,
         refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
@@ -591,7 +592,7 @@ def add_label(id, label_set, user, **kwargs):
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
 @parse_parameters(refresh=parse_refresh)
-def remove_labels(id, label_set, user, **kwargs):
+def remove_labels(id: str, label_set: str, user: User, server_version: str | None = None, **kwargs):
     """Remove labels from a hit.
 
     Variables:
@@ -632,7 +633,8 @@ def remove_labels(id, label_set, user, **kwargs):
     hit_service.update_hit(
         id,
         [hit_helper.list_remove(f"howler.labels.{label_set}", label) for label in labels],
-        user["uname"],
+        user.uname,
+        version=server_version,
         refresh="true" if DEBUG_FORCE_REFRESH else refresh,
     )
 
@@ -648,7 +650,7 @@ def remove_labels(id, label_set, user, **kwargs):
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=True)
 @parse_parameters(refresh=parse_refresh)
-def transition(id: str, user: User, **kwargs):
+def transition(id: str, user: User, server_version: str | None = None, **kwargs):
     """Transition a hit
 
     Variables:
@@ -679,11 +681,12 @@ def transition(id: str, user: User, **kwargs):
         return bad_request(err="Invalid data format")
 
     transition = transition_data["transition"]
+    version: str | None = None
     if "If-Match" in request.headers:
         version = request.headers["If-Match"]
     else:
         logger.warning("User is mising version - no If-Match header in request.")
-        version = None
+        version = server_version
 
     try:
         if transition not in HitStatusTransition.list():
@@ -694,7 +697,7 @@ def transition(id: str, user: User, **kwargs):
                 )
             )
 
-        hit_service.transition_hit(
+        hit, version = hit_service.transition_hit(
             id, transition, user, version, refresh=refresh, **kwargs, **transition_data.get("data", {})
         )
     except (WorkflowException, DataStoreException, InvalidDataException) as e:
@@ -704,7 +707,6 @@ def transition(id: str, user: User, **kwargs):
     except HowlerException as e:
         return internal_error(err=str(e))
 
-    hit, version = hit_service.get_hit(id, as_odm=False, version=True)
     return ok(hit), version
 
 
@@ -712,7 +714,7 @@ def transition(id: str, user: User, **kwargs):
 @hit_api.route("/<id>/comments/<comment_id>", methods=["GET"])
 @api_login(audit=False, required_priv=["R"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
-def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwargs):
+def get_comment(id: str, comment_id: str, user: User, server_version: str | None = None, **kwargs):
     """Get a comment associated with a particular hit
 
     Variables:
@@ -741,7 +743,7 @@ def get_comment(id: str, comment_id: str, user: User, server_version: str, **kwa
 @hit_api.route("/<id>/comments", methods=["POST"])
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
-def add_comment(id: str, user: dict[str, Any], **kwargs):
+def add_comment(id: str, user: User, server_version: str | None = None, **kwargs):
     """Add a comment
 
     Variables:
@@ -781,12 +783,13 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
             [
                 hit_helper.list_add(
                     "howler.comment",
-                    Comment({"user": user["uname"], "value": comment_value}),
+                    Comment({"user": user.uname, "value": comment_value}),
                     explanation=f"Added a comment:\n\n{comment_value}",
                     if_missing=True,
                 ),
             ],
-            user["uname"],
+            user.uname,
+            version=server_version,
         )
     except DataStoreException as e:
         return bad_request(err=str(e))
@@ -800,7 +803,7 @@ def add_comment(id: str, user: dict[str, Any], **kwargs):
 @hit_api.route("/<id>/comments/<comment_id>", methods=["PUT"])
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
-def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
+def edit_comment(id: str, comment_id: str, user: dict[str, Any], server_version: str | None = None, **kwargs):
     """Edit a comment
 
     Variables:
@@ -865,6 +868,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
             ),
         ],
         user["uname"],
+        version=server_version,
     )
 
     return ok(new_hit), version
@@ -874,7 +878,7 @@ def edit_comment(id: str, comment_id: str, user: dict[str, Any], **kwargs):
 @hit_api.route("/<id>/comments", methods=["DELETE"])
 @api_login(audit=False, required_priv=["W"])
 @add_etag(getter=hit_service.get_hit, check_if_match=False)
-def delete_comments(id: str, user: User, **kwargs):
+def delete_comments(id: str, user: User, server_version: str | None = None, **kwargs):
     """Delete a set of comments
 
     Variables:
@@ -923,6 +927,7 @@ def delete_comments(id: str, user: User, **kwargs):
                 for comment in comments
             ],
             user.uname,
+            version=server_version,
         )
 
     except DataStoreException as e:

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/cronjobs/action_queue_worker.py

@@ -50,15 +50,18 @@ def run_worker(trigger: str) -> None:  # pragma: no cover – long-running loop,
             if item is not None:
                 batch.append(item)
 
+            if len(batch) % 10 == 0 or len(batch) >= BATCH_SIZE:
+                logger.info("Batch size: %s", len(batch))
+
             if len(batch) >= BATCH_SIZE or (item is None and batch):
+                finalized_batch = [*batch]
+                batch = []
                 logger.debug("Processing action batch of %d item(s) for trigger=%s", len(batch), trigger)
                 try:
-                    process_action_batch(trigger, batch)
+                    process_action_batch(trigger, finalized_batch)
                     logger.info("Action batch complete: %d item(s) processed for trigger=%s", len(batch), trigger)
                 except Exception:
                     logger.exception("Error processing action batch for trigger=%s", trigger)
-                finally:
-                    batch = []
         except Exception:
             logger.exception("Unexpected error in action queue worker loop for trigger=%s", trigger)
 

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/datastore/collection.py

@@ -1613,6 +1613,10 @@ class ESCollection(Generic[ModelType]):
         index = self.name
         seq_no = None
         primary_term = None
+
+        if version is None and self.ilm_config:
+            _, version = self.get_if_exists(key, as_obj=False, version=True)
+
         if version:
             index, seq_no, primary_term = self._get_version_write_target(version)
 

howler_api-4.1.0.dev1430/howler/external/check_duplicates.py

@@ -0,0 +1,152 @@
+"""Find and optionally remove duplicate documents from an ILM collection."""
+
+import argparse
+import heapq
+import sys
+from collections.abc import Iterator
+from typing import Any
+
+
+def iter_index_ids(collection: Any, index: str, batch_size: int) -> Iterator[str]:
+    """Yield document IDs from *index* in sorted order without retaining prior pages."""
+    search_after = None
+
+    while True:
+        search_args = {
+            "index": index,
+            "query": {"match_all": {}},
+            "sort": [{"id": "asc"}],
+            "size": batch_size,
+            "_source": False,
+            "track_total_hits": False,
+        }
+        if search_after is not None:
+            search_args["search_after"] = search_after
+
+        response = collection.with_retries(collection.datastore.client.search, **search_args)
+        hits = response["hits"]["hits"]
+        if not hits:
+            return
+
+        for hit in hits:
+            yield hit["_id"]
+
+        search_after = hits[-1]["sort"]
+
+
+def _push_next_id(heap: list[tuple[str, str, Iterator[str]]], index: str, document_ids: Iterator[str]) -> None:
+    try:
+        heapq.heappush(heap, (next(document_ids), index, document_ids))
+    except StopIteration:
+        pass
+
+
+def iter_duplicates(collection: Any, indexes: list[str], batch_size: int) -> Iterator[tuple[str, list[str]]]:
+    """Yield each duplicated ID and the physical indexes that contain it.
+
+    This is a k-way merge over paginated, sorted searches. Its memory use is bounded by
+    the number of physical indexes and one Elasticsearch result page per active iterator.
+    """
+    heap: list[tuple[str, str, Iterator[str]]] = []
+    for index in indexes:
+        document_ids = iter_index_ids(collection, index, batch_size)
+        _push_next_id(heap, index, document_ids)
+
+    while heap:
+        document_id, index, document_ids = heapq.heappop(heap)
+        duplicate_indexes = [index]
+        _push_next_id(heap, index, document_ids)
+
+        while heap and heap[0][0] == document_id:
+            _, index, document_ids = heapq.heappop(heap)
+            duplicate_indexes.append(index)
+            _push_next_id(heap, index, document_ids)
+
+        if len(duplicate_indexes) > 1:
+            yield document_id, duplicate_indexes
+
+
+def delete_duplicates(collection: Any, document_id: str, indexes: list[str]) -> int:
+    """Delete all supplied physical-index copies of a document ID."""
+    deleted = 0
+    for index in indexes:
+        response = collection.with_retries(collection.datastore.client.delete, index=index, id=document_id)
+        if response["result"] == "deleted":
+            deleted += 1
+    return deleted
+
+
+def main() -> int:
+    """Run the interactive duplicate scan for an ILM collection."""
+    parser = argparse.ArgumentParser(
+        description="Find duplicate document IDs across an ILM collection's physical indexes."
+    )
+    parser.add_argument("collection", help="Collection name registered in the datastore.")
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=1_000,
+        help="IDs fetched from each physical index per Elasticsearch request (default: 1000).",
+    )
+    args = parser.parse_args()
+
+    if args.batch_size <= 0:
+        parser.error("--batch-size must be a positive integer.")
+
+    from howler.datastore.collection import ESCollection
+
+    # Checking duplicates must not create collections or alter their ILM configuration.
+    ESCollection.IGNORE_ENSURE_COLLECTION = True
+    ESCollection.ENSURE_COLLECTION_WARNED = True
+
+    from howler.common import loader
+
+    datastore = loader.datastore(archive_access=False)
+    collection_names = datastore.ds.get_models()
+    if args.collection not in collection_names:
+        parser.error(
+            f"Unknown collection '{args.collection}'. Valid collections: {', '.join(sorted(collection_names))}"
+        )
+
+    collection = getattr(datastore, args.collection)
+    if collection.ilm_config is None:
+        print(f"Collection '{args.collection}' is not ILM-enabled; nothing to check.")
+        return 0
+
+    collection._refresh_ilm_index_name()
+    indexes = collection.index_list_full
+    if len(indexes) < 2:
+        print(f"Collection '{args.collection}' has fewer than two physical ILM indexes; nothing to check.")
+        return 0
+
+    print(f"Scanning for duplicate IDs in: {', '.join(indexes)}")
+    duplicates_found = 0
+    copies_deleted = 0
+    index_order = {index: position for position, index in enumerate(indexes)}
+
+    try:
+        for document_id, duplicate_indexes in iter_duplicates(collection, indexes, args.batch_size):
+            duplicates_found += 1
+            keep_index = min(duplicate_indexes, key=index_order.__getitem__)
+            stale_indexes = [index for index in duplicate_indexes if index != keep_index]
+            print(
+                f"Duplicate ID '{document_id}' found in {', '.join(duplicate_indexes)}. "
+                f"Keeping the newest copy in '{keep_index}'."
+            )
+            answer = input(f"Delete copies from {', '.join(stale_indexes)}? [yes/NO] ")
+            if answer.lower() in {"y", "yes"}:
+                deleted_count = delete_duplicates(collection, document_id, stale_indexes)
+                copies_deleted += deleted_count
+                print(f"Deleted {deleted_count} copy/copies of '{document_id}'.")
+    except KeyboardInterrupt:
+        print("\nInterrupted; all completed deletions have been retained.", file=sys.stderr)
+        return 130
+    finally:
+        datastore.ds.close()
+
+    print(f"Scan complete. Found {duplicates_found} duplicate ID(s); deleted {copies_deleted} stale copy/copies.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/services/correlation_service.py

@@ -345,7 +345,7 @@ def run_worker() -> None:  # pragma: no cover – long-running loop, tested via
             if item is not None:
                 batch.append(item)
 
-            if len(batch) > 0:
+            if len(batch) % 10 == 0 or len(batch) >= BATCH_SIZE:
                 logger.info("Batch size: %s", len(batch))
 
             if len(batch) >= BATCH_SIZE or (item is None and batch):

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/howler/services/hit_service.py

@@ -549,7 +549,7 @@ def _update_hit(
     user: Optional[str] = None,
     version: Optional[str] = None,
     refresh: str | None = None,
-) -> tuple[dict[str, Any] | None, str | None]:
+) -> tuple[Hit | None, str | None]:
     """Internal function to update a hit with proper logging and event emission.
 
     This function applies update operations to a hit, automatically adding worklog entries
@@ -573,7 +573,11 @@ def _update_hit(
     if user and not isinstance(user, str):
         raise HowlerValueError("User must be of type string")
 
-    current_hit = cast(Hit, get_hit(hit_id, as_odm=True))
+    if version is None:
+        fetched_hit, version = datastore().hit.get(hit_id, as_obj=True, version=True)
+        current_hit = cast(Hit, fetched_hit)
+    else:
+        current_hit = cast(Hit, get_hit(hit_id, as_odm=True))
 
     for operation in operations:
         if not operation:
@@ -590,15 +594,16 @@ def _update_hit(
             is_list = current_hit.flat_fields()[key].multivalued
             previous_value = "list"
 
-        operation_type = ""
         if is_list:
-            if operation.operation in (
-                ESCollection.UPDATE_APPEND,
-                ESCollection.UPDATE_APPEND_IF_MISSING,
-            ):
-                operation_type = HitOperationType.APPENDED
-            else:
-                operation_type = HitOperationType.REMOVED
+            operation_type = (
+                HitOperationType.APPENDED
+                if operation.operation
+                in (
+                    ESCollection.UPDATE_APPEND,
+                    ESCollection.UPDATE_APPEND_IF_MISSING,
+                )
+                else HitOperationType.REMOVED
+            )
         else:
             operation_type = HitOperationType.SET
 
@@ -625,9 +630,9 @@ def _update_hit(
 
     datastore().hit.update(hit_id, final_operations, version, refresh=refresh)
     # Need to fetch the new data of the hit for the comms_service
-    data, _version = datastore().hit.get(hit_id, as_obj=False, version=True) or (None, None)
+    data, _version = datastore().hit.get(hit_id, as_obj=True, version=True) or (None, None)
     if data and _version:
-        comms_service.emit("hits", {"hit": data, "version": _version})
+        comms_service.emit("hits", {"hit": data.as_primitives(), "version": _version})
 
     return data, _version
 
@@ -652,7 +657,7 @@ def transition_hit(
     version: Optional[str] = None,
     refresh: str | None = None,
     **kwargs,
-):
+) -> tuple[Hit | None, str | None]:
     """Transition a hit from one status to another while updating related properties.
 
     For certain transitions (PROMOTE, DEMOTE, ASSESS, RE_EVALUATE), it also executes bulk actions and emits events.
@@ -664,6 +669,9 @@ def transition_hit(
         version: Optional version to validate against. The transition will not run if the version doesn't match.
         **kwargs: Additional arguments including potential 'hit' object and 'assessment' value
 
+    Returns:
+        A tuple containing the updated hit and the optimistic-lock version token after the transition.
+
     Raises:
         NotFoundException: If the hit does not exist
     """
@@ -684,10 +692,12 @@ def transition_hit(
 
     # Apply the workflow transition to get required updates
     updates = workflow.transition(hit_status, transition, user=user, hit=hit, **kwargs)
+    new_version = version
 
     # Apply updates if any were generated by the workflow
+    updated_hit: Hit | None = None
     if updates:
-        _update_hit(hit_id, updates, user.uname, version=version, refresh=refresh)
+        updated_hit, new_version = _update_hit(hit_id, updates, user.uname, version=version, refresh=refresh)
 
     # Execute bulk actions for transitions that require them
     # These transitions need additional processing beyond the workflow
@@ -720,8 +730,13 @@ def transition_hit(
         action_service.enqueue_action_execution([hit_id], trigger=trigger, user=user)
 
         # Emit events for processed hit to notify other systems
-        data, hit_version = datastore().hit.get(hit_id, as_obj=False, version=True)
-        comms_service.emit("hits", {"hit": data, "version": hit_version})
+        updated_hit, new_version = datastore().hit.get(hit_id, as_obj=True, version=True)
+        comms_service.emit(
+            "hits", {"hit": updated_hit.as_primitives() if updated_hit else None, "version": new_version}
+        )
+
+        updated_hit, new_version = datastore().hit.get(hit_id, as_obj=True, version=True)
+    return updated_hit, new_version
 
 
 DELETED_HITS = Counter(f"{APP_NAME.replace('-', '_')}_deleted_hits_total", "The number of deleted hits")

{howler_api-4.1.0.dev1405 → howler_api-4.1.0.dev1430}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.1.0.dev1405"
+version = "4.1.0.dev1430"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",