clue-api 1.6.0.dev359__tar.gz → 1.6.0.dev370__tar.gz

{clue_api-1.6.0.dev359 → clue_api-1.6.0.dev370}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.6.0.dev359
+Version: 1.6.0.dev370
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.6.0.dev359 → clue_api-1.6.0.dev370}/clue/services/lookup_service.py

@@ -678,6 +678,13 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
     query_sources = query_params.query_sources
     excluded_sources = query_params.excluded_sources
     available_sources = get_sources(user)
+    selected_sources = [
+        source
+        for source in available_sources
+        if (not query_sources or source.name in query_sources)
+        and source.name not in excluded_sources
+        and (query_sources or source.include_default)
+    ]
 
     logger.debug(
         f"Beginning enrichment for {len(data)} selectors on sources "
@@ -698,12 +705,20 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
         bulk_result.setdefault(entry.type, {})
         bulk_result[entry.type].setdefault(entry.value, {})
 
-    pool_size = min(len(data) * len(query_sources or available_sources), int(os.environ.get("EXECUTOR_THREADS", 32)))
-    thread_pool = Pool(pool_size)
+    if not selected_sources:
+        raise InvalidDataException("You must provide at least one source.")
 
+    pool_size = min(len(data) * len(selected_sources), int(os.environ.get("EXECUTOR_THREADS", 32)))
+    thread_pool = Pool(pool_size)
     existing_results: dict[str, list[dict[str, str]]] = {}
     if config.ui.replication:
-        existing_results = mongo_service.existing_results(user["uname"], "selectors", data, available_sources)
+        if query_params.no_cache:
+            # no_cache bypasses the mongo cache check entirely (forcing every source to be re-queried) and
+            # invalidates whatever was previously cached so the fresh results replace it, rather than piling
+            # up as duplicates alongside it.
+            mongo_service.invalidate_existing(user["uname"], "selectors", data, selected_sources)
+        else:
+            existing_results = mongo_service.existing_results(user["uname"], "selectors", data, selected_sources)
 
     sources_per_entry: dict[tuple[str, str], tuple[list[str], list[str]]] = {}
     for entry in data:
@@ -723,14 +738,7 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
                 )
 
     greenlets: list[tuple[list[Selector], ExternalSource, Greenlet[Any, dict[str, dict[str, QueryResult]]]]] = []
-    for source in available_sources:
-        if query_sources and source.name not in query_sources:
-            continue
-        if excluded_sources and source.name in excluded_sources:
-            continue
-        if not query_sources and not source.include_default:
-            continue
-
+    for source in selected_sources:
         obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
 
         if error:

{clue_api-1.6.0.dev359 → clue_api-1.6.0.dev370}/clue/services/mongo_service.py

@@ -4,7 +4,7 @@ load_dotenv()
 
 import json
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any
 
 from flask import Response
@@ -19,7 +19,7 @@ from clue.config import config, get_redis
 from clue.models.config import ExternalSource
 from clue.models.schema import get_bson_schema
 from clue.models.selector import Selector
-from clue.models.sync import ChangeRow, Checkpoint, PublishEvent, SelectorDocument
+from clue.models.sync import ChangeRow, Checkpoint, PublishEvent, SelectorDocument, generate_updated_at
 
 logger = get_logger(__file__)
 
@@ -329,9 +329,8 @@ def existing_results(
         return {}
 
     try:
-        types = [selector.type for selector in selectors]
-        values = [selector.value for selector in selectors]
         sources = [source.name for source in external_sources]
+        selector_pairs = [{"type": selector.type, "value": selector.value} for selector in selectors]
 
         raw_result = (
             _get_collection(user, collection)
@@ -339,11 +338,14 @@ def existing_results(
                 [
                     {
                         "$match": {
-                            "type": {"$in": types},
-                            "value": {"$in": values},
                             "source": {"$in": sources},
                             "_deleted": False,
                             "error": None,
+                            "$and": [
+                                {"$or": selector_pairs},
+                                # Only records with no expiry, or one that hasn't passed yet, count as "existing".
+                                {"$or": [{"expiry": None}, {"expiry": {"$gt": datetime.now(timezone.utc)}}]},
+                            ],
                         }
                     },
                     {"$group": {"_id": "$source", "records": {"$push": {"type": "$type", "value": "$value"}}}},
@@ -356,3 +358,37 @@ def existing_results(
     except ClueRuntimeError:
         # ClueRuntimeError if mongodb isn't functional - we'll silently fail so enrichment can continue
         return {}
+
+
+def invalidate_existing(
+    user: str, collection: str, selectors: list[Selector], external_sources: list[ExternalSource]
+) -> None:
+    """Mark cached documents matching the given selectors/sources as deleted.
+
+    Used when a caller requests `no_cache`, so the stale cached results don't stick around
+    (as duplicates) once the freshly queried results are replicated back into the collection.
+
+    Args:
+        user (str): The username to use as the collection name.
+        collection (str): The collection name within the user's database.
+        selectors (list[Selector]): List of selectors (type/value pairs) to invalidate.
+        external_sources (list[ExternalSource]): List of external sources to filter by.
+    """
+    if not config.ui.replication:
+        return
+
+    try:
+        sources = [source.name for source in external_sources]
+        selector_pairs = [{"type": selector.type, "value": selector.value} for selector in selectors]
+
+        _get_collection(user, collection).update_many(
+            {
+                "source": {"$in": sources},
+                "_deleted": False,
+                "$or": selector_pairs,
+            },
+            {"$set": {"_deleted": True, "updated_at": generate_updated_at()}},
+        )
+    except ClueRuntimeError:
+        # ClueRuntimeError if mongodb isn't functional - we'll silently fail so enrichment can continue
+        pass

{clue_api-1.6.0.dev359 → clue_api-1.6.0.dev370}/pyproject.toml

@@ -141,7 +141,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.6.0.dev359"
+version = "1.6.0.dev370"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"