clue-api 1.6.0.dev323__tar.gz → 1.6.0.dev337__tar.gz

{clue_api-1.6.0.dev323 → clue_api-1.6.0.dev337}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.6.0.dev323
+Version: 1.6.0.dev337
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.6.0.dev323 → clue_api-1.6.0.dev337}/clue/api/v1/lookup.py

@@ -90,7 +90,10 @@ def bulk_enrich(**kwargs) -> dict[str, dict[str, dict[str, QueryResult]]]:
 
     Optional Arguments:
     classification: string  => Classification of the type [Default: minimum configured classification]
-    sources: string         => | separated list of data sources. If empty, all configured sources are used.
+    sources: string         => | separated list of data sources.
+        A source prefixed with '-' will be excluded. Exclusion takes precedence over inclusion.
+        If sources is empty or only exclusions, all default configured sources are used (with exclusions applied).
+        Note, a source list that includes and excludes the same sources (e.g. sources=vt|-vt) is not treated as empty.
     max_timeout: number     => Maximum execution time for the call in seconds
     limit: number           => limit the amount of returned results counted per source
     no_annotation: boolean  => Do not return any anotations
@@ -171,7 +174,10 @@ def enrich(type_name: str, value: str, **kwargs) -> dict[str, QueryResult]:
 
     Optional Arguments:
     classification: string  => Classification of the type [Default: minimum configured classification]
-    sources: string         => | separated list of data sources. If empty, all configured sources are used.
+    sources: string         => | separated list of data sources.
+        A source prefixed with '-' will be excluded. Exclusion takes precedence over inclusion.
+        If sources is empty or only exclusions, all default configured sources are used (with exclusions applied).
+        Note, a source list that includes and excludes the same sources (e.g. sources=vt|-vt) is not treated as empty.
     max_timeout: number     => Maximum execution time for the call in seconds
     limit: number           => limit the amount of returned results counted per source
     no_annotation: boolean  => Do not return any anotations

{clue_api-1.6.0.dev323 → clue_api-1.6.0.dev337}/clue/services/fetcher_service.py

@@ -3,7 +3,7 @@ from urllib.parse import urljoin
 
 import elasticapm
 import requests
-from flask import request
+from flask import has_request_context, request
 from pydantic import TypeAdapter, ValidationError
 from requests import JSONDecodeError, exceptions
 
@@ -11,6 +11,7 @@ from clue.common.exceptions import (
     AuthenticationException,
     ClueException,
     ClueValueError,
+    InvalidDataException,
     NotFoundException,
 )
 from clue.common.logging import get_logger
@@ -26,6 +27,26 @@ logger = get_logger(__file__)
 CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
 
 
+def get_obo_access_token(
+    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+) -> tuple[Optional[str], Optional[str]]:
+    """Get the caller access token and an OBO token for an external source when needed."""
+    if access_token is None and has_request_context():
+        auth_header = request.headers.get("Authorization", type=str)
+        if auth_header:
+            parts = auth_header.split(" ", 1)
+            access_token = parts[1] if len(parts) == 2 and parts[0].lower() == "bearer" else auth_header
+    if not access_token:
+        return None, None
+
+    obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
+    if error:
+        logger.error("%s: %s", source.name, error)
+        raise AuthenticationException("Invalid token provided for this enrichment.")
+
+    return access_token, obo_access_token
+
+
 @cache.memoize(timeout=1 if DEBUG else 5 * 60, args_to_ignore=["access_token"])  # Cached for 5 minutes
 def get_supported_fetchers(
     source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
@@ -43,13 +64,10 @@ def get_supported_fetchers(
 
     url = urljoin(source.url, "fetchers/")
 
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
-
-        if error:
-            logger.error("%s: %s", source.name, error)
-            return {}
+    try:
+        access_token, obo_access_token = get_obo_access_token(source, user, access_token)
+    except AuthenticationException:
+        return {}
 
     headers = {"Accept": "application/json"}
     if obo_access_token or access_token:
@@ -130,6 +148,15 @@ def get_plugins_supported_fetchers(user: dict[str, Any]) -> dict[str, FetcherDef
     return available_fetchers
 
 
+def _validate_fetcher_classification(fetcher: FetcherDefinition | None, selector: Selector, fetcher_id: str) -> None:
+    if fetcher and not CLASSIFICATION.is_accessible(fetcher.classification, selector.classification):
+        raise InvalidDataException(
+            f"Cannot send data classified as {selector.classification} to fetcher {fetcher_id} "
+            f"at classification {fetcher.classification}.",
+            status_code=400,
+        )
+
+
 def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> FetcherResult:
     """Executes a specified fetcher.
 
@@ -150,30 +177,34 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
     if not plugin:
         raise NotFoundException(f"Plugin {plugin_id} does not exist.")
 
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
-
-        if error:
-            logger.error("%s: %s", plugin.name, error)
-            raise AuthenticationException("Invalid token provided for this enrichment.")
+    access_token, obo_access_token = get_obo_access_token(plugin, user)
 
     headers = {"Accept": "application/json"}
     if obo_access_token or access_token:
         headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
 
-    if request.content_type == "application/json":
+    if request.is_json:
         parameters = request.json
     else:
-        # TODO: Pass parameters via urlencode?
-        parameters = {}
+        logger.error(
+            "Invalid content-type detected: %s",
+        )
+        raise ClueValueError(
+            "The request body must be of type application/json.",
+            status_code=400,
+        )
 
     try:
-        Selector.model_validate(parameters)
+        selector = Selector.model_validate(parameters)
+        supported_fetchers = get_supported_fetchers(plugin, user, access_token=access_token)
+
+        if len(supported_fetchers) < 1:
+            raise NotFoundException(f"{plugin_id} does not support any fetchers.")
+
+        fetcher = supported_fetchers.get(fetcher_id)
+        if fetcher is None:
+            raise NotFoundException(f"Fetcher {fetcher_id} does not exist", status_code=404)
+        _validate_fetcher_classification(fetcher, selector, fetcher_id)
 
         response = requests.post(
             urljoin(plugin.url, f"fetchers/{fetcher_id}"),
@@ -192,7 +223,7 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
         return FetcherResult.model_validate(result["api_response"], context={"is_response": True})
     except ValidationError as err:
         logger.exception("Invalid Request Body:")
-        raise ClueValueError(
+        raise InvalidDataException(
             "Validation error encountered on request body. Ensure your request body is properly formatted.",
             status_code=400,
         ) from err
@@ -224,17 +255,7 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict
     if not plugin:
         raise NotFoundException(f"Plugin {plugin_id} does not exist.")
 
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
-
-        if error:
-            logger.error("%s: %s", plugin.name, error)
-            raise AuthenticationException("Invalid token provided for this enrichment.")
+    access_token, obo_access_token = get_obo_access_token(plugin, user)
 
     headers = {"Accept": "application/json"}
     if obo_access_token or access_token:

{clue_api-1.6.0.dev323 → clue_api-1.6.0.dev337}/clue/services/lookup_service.py

@@ -107,6 +107,7 @@ class ParsedParams(BaseModel):
     "Validation of parameters parsed from request"
 
     query_sources: list[str]
+    excluded_sources: list[str]
     max_timeout: float
     limit: int
     type_classification: str
@@ -153,8 +154,11 @@ def parse_query_params(request: Request, limit: int = 10, timeout: float = 5.0):
     else:
         query_sources = []
 
+    include_sources, exclude_sources = _parse_source_list(query_sources)
+
     return ParsedParams(
-        query_sources=query_sources,
+        query_sources=include_sources,
+        excluded_sources=exclude_sources,
         max_timeout=parse_timeout(timeout),
         limit=limit,
         type_classification=type_classification,
@@ -456,6 +460,7 @@ def enrich(type_name: str, value: str, user: dict[str, Any]):  # noqa: C901
     """
     query_params = parse_query_params(request=request)
     query_sources = query_params.query_sources
+    excluded_sources = query_params.excluded_sources
     available_sources = get_sources(user)
 
     access_token = request.headers.get("Authorization", type=str)
@@ -465,7 +470,8 @@ def enrich(type_name: str, value: str, user: dict[str, Any]):  # noqa: C901
 
     logger.debug(
         f"Beginning enrichment for single selector on sources "
-        f"[{','.join(query_sources or [source.name for source in available_sources])}]"
+        f"[{','.join(query_sources or [source.name for source in available_sources if source.include_default])}] "
+        f"excluding sources [{','.join(excluded_sources)}]"
     )
 
     results: dict[str, QueryResult] = {}
@@ -478,7 +484,9 @@ def enrich(type_name: str, value: str, user: dict[str, Any]):  # noqa: C901
     for source in available_sources:
         if query_sources and source.name not in query_sources:
             continue
-        elif not query_sources and not source.include_default:
+        if excluded_sources and source.name in excluded_sources:
+            continue
+        if not query_sources and not source.include_default:
             continue
 
         finish_result = functools.partial(build_result, type_name, value, source)
@@ -668,11 +676,13 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
     """create searches for external sources"""
     query_params = parse_query_params(request=request)
     query_sources = query_params.query_sources
+    excluded_sources = query_params.excluded_sources
     available_sources = get_sources(user)
 
     logger.debug(
         f"Beginning enrichment for {len(data)} selectors on sources "
-        f"[{','.join(query_sources or [source.name for source in available_sources])}]"
+        f"[{','.join(query_sources or [source.name for source in available_sources if source.include_default])}] "
+        f"excluding sources [{','.join(excluded_sources)}]"
     )
 
     access_token = request.headers.get("Authorization", type=str)
@@ -695,11 +705,30 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
     if config.ui.replication:
         existing_results = mongo_service.existing_results(user["uname"], "selectors", data, available_sources)
 
+    sources_per_entry: dict[tuple[str, str], tuple[list[str], list[str]]] = {}
+    for entry in data:
+        if entry.sources is not None:
+            entry_key = (entry.type, entry.value)
+            if entry_key not in sources_per_entry:
+                sources_per_entry[entry_key] = _parse_source_list(entry.sources)
+
+            else:
+                logger.warning("Duplicate sources found for entry %s:%s. Merging sources.", entry.type, entry.value)
+
+                existing_include_list, existing_exclude_list = sources_per_entry[entry_key]
+                include_sources, exclude_sources = _parse_source_list(entry.sources)
+                sources_per_entry[entry_key] = (
+                    list(set(existing_include_list + include_sources)),
+                    list(set(existing_exclude_list + exclude_sources)),
+                )
+
     greenlets: list[tuple[list[Selector], ExternalSource, Greenlet[Any, dict[str, dict[str, QueryResult]]]]] = []
     for source in available_sources:
         if query_sources and source.name not in query_sources:
             continue
-        elif not query_sources and not source.include_default:
+        if excluded_sources and source.name in excluded_sources:
+            continue
+        if not query_sources and not source.include_default:
             continue
 
         obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
@@ -720,9 +749,15 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
         # check query against the max supported classification of the external system
         # if this is not supported, we should let the user know.
         for entry in data:
-            if entry.sources is not None and source.name not in entry.sources:
+            if entry.sources == []:  # support old behaviour of empty entry sources -> use no sources
                 continue
 
+            parsed_sources = sources_per_entry.get((entry.type, entry.value))
+            if parsed_sources is not None:
+                include_sources, exclude_sources = parsed_sources
+                if (include_sources and source.name not in include_sources) or source.name in exclude_sources:
+                    continue
+
             if (
                 source.name in existing_results
                 and {"type": entry.type, "value": entry.value} in existing_results[source.name]
@@ -802,3 +837,30 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
     thread_pool.kill(block=False)
 
     return bulk_result
+
+
+def _parse_source_list(source_list: list[str]) -> tuple[list[str], list[str]]:
+    """Splits the provided source list into include and exclude lists.
+
+    Args:
+        source_list (list[str]): The list of sources to parse.
+
+    Returns:
+        tuple[list[str], list[str]]: (include_sources, exclude_sources)
+            The list of sources to include and the list of sources to exclude.
+    """
+    include_sources = []
+    exclude_sources = []
+
+    for source in source_list:
+        source = source.strip()
+        if not source:
+            continue
+        if source.startswith("-"):
+            excluded_source = source[1:].strip()
+            if excluded_source:
+                exclude_sources.append(excluded_source)
+        else:
+            include_sources.append(source)
+
+    return include_sources, exclude_sources

{clue_api-1.6.0.dev323 → clue_api-1.6.0.dev337}/pyproject.toml

@@ -141,7 +141,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.6.0.dev323"
+version = "1.6.0.dev337"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"