clue-api 1.7.0.dev433__tar.gz → 1.7.0.dev435__tar.gz

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.7.0.dev433
+Version: 1.7.0.dev435
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/api/v1/actions.py

@@ -8,7 +8,7 @@ List and execute actions
 
 from flask_cors import CORS
 
-from clue.api import internal_error, make_subapi_blueprint, not_found, ok
+from clue.api import internal_error, make_subapi_blueprint, not_found, ok, service_unavailable
 from clue.common.exceptions import ClueException, NotFoundException
 from clue.common.logging import get_logger
 from clue.common.swagger import generate_swagger_docs
@@ -22,7 +22,7 @@ logger = get_logger(__file__)
 
 SUB_API = "actions"
 actions_api = make_subapi_blueprint(SUB_API, api_version=1)
-actions_api._doc = "Run actions on data through configured external data sources/systems."
+actions_api._doc = "Run actions on data through configured external data sources/systems."  # type: ignore
 
 CORS(actions_api, origins=config.ui.cors_origins, supports_credentials=True)
 
@@ -122,4 +122,6 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, **kwargs) ->
     except NotFoundException as err:
         return not_found(err=err.message)
     except ClueException as err:
+        if err.status_code == 503:
+            return service_unavailable(err=err.message)
         return internal_error(err=err.message)

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/api/v1/fetchers.py

@@ -8,7 +8,7 @@ List and execute fetchers that provide data to be rendered client-side
 
 from flask_cors import CORS
 
-from clue.api import bad_gateway, bad_request, make_subapi_blueprint, not_found, ok
+from clue.api import bad_gateway, bad_request, make_subapi_blueprint, not_found, ok, service_unavailable
 from clue.common.exceptions import ClueException, NotFoundException
 from clue.common.logging import get_logger
 from clue.common.swagger import generate_swagger_docs
@@ -120,6 +120,8 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, **kwargs):
     except NotFoundException as err:
         return not_found(err=err.message)
     except ClueException as err:
+        if err.status_code == 503:
+            return service_unavailable(err=err.message)
         if err.status_code == 400:
             logger.warning(
                 "Bad request from fetcher %s.%s with task_id: %s: %s", plugin_id, fetcher_id, task_id, err.message

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/helper/headers.py

@@ -1,11 +1,10 @@
 from clue.common.logging import get_logger
-from clue.config import DEBUG, cache, config
+from clue.config import config
 
 logger = get_logger(__file__)
 
 
-@cache.memoize(timeout=1 if DEBUG else 5 * 60)  # Cached for 5 minutes
-def generate_headers(access_token: str | None, clue_access_token: str | None) -> dict[str, str]:
+def generate_headers(obo_access_token: str | None, access_token: str | None) -> dict[str, str]:
     """Generates the request headers.
 
     Args:
@@ -19,12 +18,12 @@ def generate_headers(access_token: str | None, clue_access_token: str | None) ->
         "content-type": "application/json",
     }
 
-    if access_token:
+    if obo_access_token or access_token:
         logger.debug("Appending authorization header")
-        _headers["Authorization"] = f"Bearer {access_token}"
+        _headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
 
-    if config.auth.propagate_clue_key and clue_access_token:
+    if config.auth.propagate_clue_key and obo_access_token and access_token:
         logger.debug("Appending custom authorization header")
-        _headers["X-Clue-Authorization"] = clue_access_token
+        _headers["X-Clue-Authorization"] = access_token
 
     return _headers

clue_api-1.7.0.dev435/clue/helper/obo.py

@@ -0,0 +1,31 @@
+from typing import Any, Optional
+
+from flask import has_request_context, request
+
+from clue.common.exceptions import AuthenticationException
+from clue.common.logging import get_logger
+from clue.models.config import ExternalSource
+from clue.services import auth_service
+
+logger = get_logger(__file__)
+
+
+def get_obo_access_token(
+    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+) -> tuple[Optional[str], Optional[str]]:
+    """Get the caller access token and an OBO token for an external source when needed."""
+    if access_token is None and has_request_context():
+        auth_header = request.headers.get("Authorization", type=str)
+        if auth_header:
+            parts = auth_header.split(" ", 1)
+            access_token = parts[1] if len(parts) == 2 else auth_header
+
+    if not access_token:
+        return None, None
+
+    obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
+    if error:
+        logger.error("%s: %s", source.name, error)
+        raise AuthenticationException("Invalid token provided for this enrichment.")
+
+    return access_token, obo_access_token

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/services/action_service.py

@@ -1,80 +1,76 @@
-from typing import Any, Optional
+from time import monotonic
+from typing import Any
 from urllib.parse import urljoin
 
 import requests
 from elasticapm.traces import capture_span
 from flask import request
-from pydantic import TypeAdapter, ValidationError
+from pydantic import TypeAdapter
 from requests import JSONDecodeError, exceptions
 
-from clue.common.exceptions import ClueException, NotFoundException
+from clue.common.exceptions import AuthenticationException, ClueException, NotFoundException
 from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION, config
 from clue.helper.headers import generate_headers
+from clue.helper.obo import get_obo_access_token
 from clue.helper.plugin_requests import request_with_safe_redirects
 from clue.models.actions import ActionResult, ActionSpec
 from clue.models.config import ExternalSource
-from clue.services import auth_service
 
 logger = get_logger(__file__)
 
 
+def _raise_action_metadata_unavailable(error: Exception | None = None) -> None:
+    raise ClueException(
+        "Unable to verify action availability with the upstream source.", error, status_code=503
+    ) from error
+
+
 def get_supported_actions(
-    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+    source: ExternalSource,
+    headers: dict[str, str],
+    *,
+    timeout: float = 10.0,
+    raise_on_error: bool = False,
 ) -> dict[str, ActionSpec]:
     """Gets all supported actions for a source
 
     Args:
-        source_url (str): The URL of the source
-        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+        source (ExternalSource): The source whose actions to retrieve.
+        headers (dict[str, str]): Headers to include in the upstream request.
+        timeout (float): The upstream request timeout in seconds.
+        raise_on_error (bool): Raise a 503 when metadata cannot be verified.
 
     Returns:
         dict[str, ActionSpec]: A dict of each action and their schema
     """
     logger.info("Fetching actions for source %s", source.name)
-
     url = urljoin(source.url, "actions/")
 
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
-
-        if error:
-            logger.error("%s: %s", source.name, error)
-            return {}
-
-    headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
-
     with capture_span(f"GET {url}", span_type="http"):
         rsp = None
         try:
-            rsp = request_with_safe_redirects(requests.get, url, headers=headers, timeout=10.0)
+            rsp = request_with_safe_redirects(requests.get, url, headers=headers, timeout=timeout)
             result = rsp.json()
 
             if not rsp.ok:
                 err = result["api_error_message"]
                 logger.error(f"Error from upstream server: {rsp.status_code=}, {err=}")
+                if raise_on_error:
+                    _raise_action_metadata_unavailable()
+                return {}
 
             return TypeAdapter(dict[str, ActionSpec]).validate_python(result["api_response"])
-        except (exceptions.ConnectionError, exceptions.Timeout):
-            # any errors are logged and no result is saved to local cache to enable retry on next query
-            logger.exception("Unable to connect: %s", url)
-            return {}
-        except (requests.exceptions.JSONDecodeError, KeyError, JSONDecodeError):
-            logger.exception(
-                "External API did not return expected format. Full data:\n\n%s\n\nStack Trace:",
-                rsp.text if rsp else "None",
-            )
-            return {}
-        except ValidationError:
-            logger.exception("ValidationError in response from %s:\n%s", source.url)
-            return {}
-        except Exception:
-            logger.exception("Unknown exception occurred on action fetching:")
+        except ClueException:
+            raise
+        except Exception as err:
+            logger.exception("Unable to retrieve action metadata from %s", source.url)
+            if raise_on_error:
+                _raise_action_metadata_unavailable(err)
             return {}
 
 
-def all_supported_actions(user: dict[str, Any], access_token: Optional[str] = None) -> dict[str, ActionSpec]:
+def all_supported_actions(user: dict[str, Any]) -> dict[str, ActionSpec]:
     """Gets all supported actions for all sources
 
     Args:
@@ -86,7 +82,17 @@ def all_supported_actions(user: dict[str, Any], access_token: Optional[str] = No
     all_actions: dict[str, ActionSpec] = {}
 
     for source in config.api.external_sources:
-        supported_actions = get_supported_actions(source, user, access_token=access_token)
+        if not CLASSIFICATION.is_accessible(user["classification"], source.classification):
+            continue
+
+        try:
+            access_token, obo_access_token = get_obo_access_token(source, user)
+        except AuthenticationException:
+            continue
+
+        supported_actions = get_supported_actions(
+            source, generate_headers(obo_access_token=obo_access_token, access_token=access_token)
+        )
         total_actions = 0
         for key, action in supported_actions.items():
             total_actions += 1
@@ -100,14 +106,7 @@ def get_plugins_supported_actions(user: dict[str, Any]) -> dict[str, ActionSpec]
     """Return the supported actions of each external service, filtered to what the user has access to."""
     available_actions: dict[str, ActionSpec] = {}
 
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    all_actions = all_supported_actions(
-        user,
-        access_token=access_token,
-    )
+    all_actions = all_supported_actions(user)
 
     logger.info("Fetching actions for classification %s", user["classification"])
 
@@ -144,22 +143,19 @@ def execute_action(plugin_id: str, action_id: str, user: dict[str, Any]) -> Acti
     """
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
-    if not plugin:
-        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
-
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
+    if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
+        raise NotFoundException("Action not found.", status_code=404)
 
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
+    try:
+        access_token, obo_access_token = get_obo_access_token(plugin, user)
+    except AuthenticationException:
+        return ActionResult(outcome="failure", summary="Invalid token provided for this enrichment.")
 
-        if error:
-            logger.error("%s: %s", plugin.name, error)
-            return ActionResult(outcome="failure", summary="Invalid token provided for this enrichment.")
+    headers = generate_headers(obo_access_token=obo_access_token, access_token=access_token)
 
-    headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
+    action = get_supported_actions(plugin, headers).get(action_id)
+    if action is None or not CLASSIFICATION.is_accessible(user["classification"], action.classification):
+        raise NotFoundException("Action not found.", status_code=404)
 
     if request.content_type == "application/json":
         parameters = request.json
@@ -211,22 +207,29 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, user: dict[s
     """
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
-    if not plugin:
-        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
-
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    obo_access_token = None
-    if access_token:
-        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
+    if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
+        raise NotFoundException("Action not found.", status_code=404)
 
-        if error:
-            logger.error("%s: %s", plugin.name, error)
-            return ActionResult(outcome="failure", summary="Invalid token provided.")
-
-    headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
+    try:
+        access_token, obo_access_token = get_obo_access_token(plugin, user)
+    except AuthenticationException:
+        return ActionResult(outcome="failure", summary="Invalid token provided.")
+
+    headers = generate_headers(obo_access_token=obo_access_token, access_token=access_token)
+
+    timeout = request.args.get("max_timeout", plugin.default_timeout, type=float)
+    metadata_started = monotonic()
+    # Authorization metadata must stay fresh; include this lookup in the caller's timeout budget.
+    action = get_supported_actions(
+        plugin,
+        headers,
+        timeout=max(min(timeout, 10.0), 0.001),
+        raise_on_error=True,
+    ).get(action_id)
+    if action is None or not CLASSIFICATION.is_accessible(user["classification"], action.classification):
+        raise NotFoundException("Action not found.", status_code=404)
+
+    remaining_timeout = max(timeout - (monotonic() - metadata_started), 0.001)
 
     try:
         req_url = urljoin(plugin.url, f"actions/{action_id}/status/{task_id}")
@@ -236,7 +239,7 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, user: dict[s
             requests.get,
             req_url,
             headers=headers,
-            timeout=request.args.get("max_timeout", plugin.default_timeout, type=float),
+            timeout=remaining_timeout,
         )
 
         result = response.json()

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/services/auth_service.py

@@ -283,7 +283,7 @@ def basic_auth(data: str, is_base64: bool = True) -> AuthResult:
 
 
 # TODO: sa-clue support
-def check_obo(source: ExternalSource, access_token: str, username: str) -> tuple[Optional[str], Optional[str]]:
+def check_obo(source: ExternalSource, access_token: str | None, username: str) -> tuple[Optional[str], Optional[str]]:
     """Checks whether a token's audience matches the source, and if it doesn't, tries to get an OBO token for the source
 
     Args:
@@ -295,16 +295,16 @@ def check_obo(source: ExternalSource, access_token: str, username: str) -> tuple
         tuple[Optional[str], Optional[str]]: A tuple with either the valid access token in the first arg, or the error
             message in the second arg.
     """
-    obo_access_token: Optional[str] = None
+    obo_access_token: str | None = None
     if source.obo_target:
         logger.debug("Source %s requires OBO to %s", source.name, source.obo_target)
 
-        if "." not in access_token:
+        if access_token and "." not in access_token:
             logger.warning("JWT not provided, using service account.")
-            if not (sa_token := jwt_service.fetch_sa_token()):
-                return None, None
+            access_token = jwt_service.fetch_sa_token()
 
-            access_token = sa_token
+        if not access_token:
+            return None, "Valid access token not provided."
 
         audience = jwt_service.extract_audience(access_token)
 

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/services/fetcher_service.py

@@ -1,9 +1,10 @@
-from typing import Any, Optional
+from time import monotonic
+from typing import Any
 from urllib.parse import urljoin
 
 import requests
 from elasticapm.traces import capture_span
-from flask import has_request_context, request
+from flask import request
 from pydantic import TypeAdapter, ValidationError
 from requests import JSONDecodeError, exceptions
 
@@ -15,48 +16,37 @@ from clue.common.exceptions import (
     NotFoundException,
 )
 from clue.common.logging import get_logger
-from clue.config import CLASSIFICATION, DEBUG, cache, config
+from clue.config import CLASSIFICATION, config
+from clue.helper.headers import generate_headers
+from clue.helper.obo import get_obo_access_token
 from clue.helper.plugin_requests import request_with_safe_redirects
 from clue.models.config import ExternalSource
 from clue.models.fetchers import FetcherDefinition, FetcherResult
 from clue.models.selector import Selector
-from clue.services import auth_service
 
 logger = get_logger(__file__)
 
-# Either cache for one second in debug mode, or five minutes in production
-CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
 
+def _raise_fetcher_metadata_unavailable(error: Exception | None = None) -> None:
+    raise ClueException(
+        "Unable to verify fetcher availability with the upstream source.", error, status_code=503
+    ) from error
 
-def get_obo_access_token(
-    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
-) -> tuple[Optional[str], Optional[str]]:
-    """Get the caller access token and an OBO token for an external source when needed."""
-    if access_token is None and has_request_context():
-        auth_header = request.headers.get("Authorization", type=str)
-        if auth_header:
-            parts = auth_header.split(" ", 1)
-            access_token = parts[1] if len(parts) == 2 and parts[0].lower() == "bearer" else auth_header
-    if not access_token:
-        return None, None
 
-    obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
-    if error:
-        logger.error("%s: %s", source.name, error)
-        raise AuthenticationException("Invalid token provided for this enrichment.")
-
-    return access_token, obo_access_token
-
-
-@cache.memoize(timeout=1 if DEBUG else 5 * 60, args_to_ignore=["access_token"])  # Cached for 5 minutes
 def get_supported_fetchers(
-    source: ExternalSource, user: dict[str, Any], access_token: Optional[str] = None
+    source: ExternalSource,
+    headers: dict[str, str],
+    *,
+    timeout: float = 5.0,
+    raise_on_error: bool = False,
 ) -> dict[str, FetcherDefinition]:
-    """Gets all supported fetchers for a source
+    """Fetch current metadata without caching authorization classifications.
 
     Args:
-        source_url (str): The URL of the source
-        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+        source (ExternalSource): The source whose fetchers to retrieve.
+        headers (dict[str, str]): Headers to include in the upstream request.
+        timeout (float): The upstream request timeout in seconds.
+        raise_on_error (bool): Raise a 503 when metadata cannot be verified.
 
     Returns:
         dict[str, FetcherDefinition]: A dict of each ids mapped to fetcher metadata
@@ -65,42 +55,33 @@ def get_supported_fetchers(
 
     url = urljoin(source.url, "fetchers/")
 
-    try:
-        access_token, obo_access_token = get_obo_access_token(source, user, access_token)
-    except AuthenticationException:
-        return {}
-
-    headers = {"Accept": "application/json"}
-    if obo_access_token or access_token:
-        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
-
     with capture_span(f"GET {url}", span_type="http"):
         try:
-            rsp = request_with_safe_redirects(requests.get, url, headers=headers, timeout=5.0)
+            rsp = request_with_safe_redirects(requests.get, url, headers=headers, timeout=timeout)
             result = rsp.json()
 
             if not rsp.ok:
                 err = result["api_error_message"]
                 logger.error(f"Error from upstream server: {rsp.status_code=}, {err=}")
+                if raise_on_error:
+                    _raise_fetcher_metadata_unavailable()
+                return {}
 
             return TypeAdapter(dict[str, FetcherDefinition]).validate_python(result["api_response"])
-        except (exceptions.ConnectionError, exceptions.Timeout):
-            # any errors are logged and no result is saved to local cache to enable retry on next query
-            logger.exception("Unable to connect: %s", url)
-            return {}
-        except (requests.exceptions.JSONDecodeError, KeyError):
-            logger.exception("External API did not return expected format:")
-            return {}
-        except ValidationError:
-            logger.exception("ValidationError in response from %s:", source.url)
+        except ClueException:
+            raise
+        except Exception as err:
+            logger.exception("Unable to retrieve fetcher metadata from %s", source.url)
+            if raise_on_error:
+                _raise_fetcher_metadata_unavailable(err)
             return {}
 
 
-def all_supported_fetchers(user: dict[str, Any], access_token: Optional[str] = None) -> dict[str, FetcherDefinition]:
+def all_supported_fetchers(user: dict[str, Any]) -> dict[str, FetcherDefinition]:
     """Gets all supported fetchers for all sources
 
     Args:
-        access_token (Optional[str], optional): The access token to use, if necessary. Defaults to None.
+        user (dict[str, Any]): The user requesting the fetchers.
 
     Returns:
         dict[str, FetcherDefinition]: A dict of all fetchers and their matching schema
@@ -108,7 +89,17 @@ def all_supported_fetchers(user: dict[str, Any], access_token: Optional[str] = N
     all_fetchers: dict[str, FetcherDefinition] = {}
 
     for source in config.api.external_sources:
-        supported_fetchers = get_supported_fetchers(source, user, access_token=access_token)
+        if not CLASSIFICATION.is_accessible(user["classification"], source.classification):
+            continue
+
+        try:
+            access_token, obo_access_token = get_obo_access_token(source, user)
+        except AuthenticationException:
+            continue
+
+        supported_fetchers = get_supported_fetchers(
+            source, generate_headers(obo_access_token=obo_access_token, access_token=access_token)
+        )
         total_fetchers = 0
         for key, action in supported_fetchers.items():
             total_fetchers += 1
@@ -122,14 +113,7 @@ def get_plugins_supported_fetchers(user: dict[str, Any]) -> dict[str, FetcherDef
     """Return the supported fetchers of each external service, filtered to what the user has access to."""
     available_fetchers: dict[str, FetcherDefinition] = {}
 
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    all_fetchers = all_supported_fetchers(
-        user,
-        access_token=access_token,
-    )
+    all_fetchers = all_supported_fetchers(user)
 
     logger.info("Retrieving fetchers for classification %s", user["classification"])
 
@@ -175,14 +159,12 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
     """
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
-    if not plugin:
-        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
+    if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
+        raise NotFoundException("Fetcher not found.", status_code=404)
 
     access_token, obo_access_token = get_obo_access_token(plugin, user)
 
-    headers = {"Accept": "application/json"}
-    if obo_access_token or access_token:
-        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
+    headers = generate_headers(obo_access_token=obo_access_token, access_token=access_token)
 
     if request.is_json:
         parameters = request.json
@@ -197,14 +179,11 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
 
     try:
         selector = Selector.model_validate(parameters)
-        supported_fetchers = get_supported_fetchers(plugin, user, access_token=access_token)
-
-        if len(supported_fetchers) < 1:
-            raise NotFoundException(f"{plugin_id} does not support any fetchers.")
+        supported_fetchers = get_supported_fetchers(plugin, headers)
 
         fetcher = supported_fetchers.get(fetcher_id)
-        if fetcher is None:
-            raise NotFoundException(f"Fetcher {fetcher_id} does not exist", status_code=404)
+        if fetcher is None or not CLASSIFICATION.is_accessible(user["classification"], fetcher.classification):
+            raise NotFoundException("Fetcher not found.", status_code=404)
         _validate_fetcher_classification(fetcher, selector, fetcher_id)
 
         response = request_with_safe_redirects(
@@ -255,14 +234,26 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict
     """
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
-    if not plugin:
-        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
+    if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
+        raise NotFoundException("Fetcher not found.", status_code=404)
 
     access_token, obo_access_token = get_obo_access_token(plugin, user)
 
-    headers = {"Accept": "application/json"}
-    if obo_access_token or access_token:
-        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
+    headers = generate_headers(obo_access_token=obo_access_token, access_token=access_token)
+
+    timeout = request.args.get("max_timeout", 60.0, type=float)
+    metadata_started = monotonic()
+    # Authorization metadata must stay fresh; include this lookup in the caller's timeout budget.
+    fetcher = get_supported_fetchers(
+        plugin,
+        headers,
+        timeout=max(min(timeout, 5.0), 0.001),
+        raise_on_error=True,
+    ).get(fetcher_id)
+    if fetcher is None or not CLASSIFICATION.is_accessible(user["classification"], fetcher.classification):
+        raise NotFoundException("Fetcher not found.", status_code=404)
+
+    remaining_timeout = max(timeout - (monotonic() - metadata_started), 0.001)
 
     try:
         req_url = urljoin(plugin.url, f"fetchers/{fetcher_id}/status/{task_id}")
@@ -272,7 +263,7 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict
             requests.get,
             req_url,
             headers=headers,
-            timeout=request.args.get("max_timeout", 60.0, type=float),
+            timeout=remaining_timeout,
         )
 
         result = response.json()

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/services/lookup_service.py

@@ -26,11 +26,12 @@ from clue.common.logging.audit import audit
 from clue.config import CLASSIFICATION as CLASSIFICATION
 from clue.config import DEBUG, config
 from clue.helper.headers import generate_headers
+from clue.helper.obo import get_obo_access_token
 from clue.helper.plugin_requests import request_with_safe_redirects
 from clue.models.config import ExternalSource
 from clue.models.network import QueryEntry, QueryResult
 from clue.models.selector import Selector
-from clue.services import auth_service, mongo_service, type_service, user_service
+from clue.services import mongo_service, type_service, user_service
 
 logger = get_logger(__file__)
 CLIENTS: dict[str, Session] = {}
@@ -361,8 +362,8 @@ def query_external(
     value: str,
     limit: int,
     timeout: float,
-    access_token: str,
-    clue_access_token: str | None,
+    access_token: str | None,
+    obo_access_token: str | None,
     no_annotation: bool = False,
     no_cache: bool = False,
     include_raw: bool = True,
@@ -375,7 +376,10 @@ def query_external(
     finish_result = functools.partial(build_result, type_name, value, source)
 
     with capture_span(query_external.__name__, span_type="greenlet"):
-        if type_name not in type_service.all_supported_types(user, access_token=access_token).get(source.name, {}):
+        supported_types = type_service.get_supported_types(
+            source.url, access_token=access_token, obo_access_token=obo_access_token
+        )
+        if not supported_types or type_name not in supported_types:
             return finish_result(error="invalid_type")
 
         if config.api.audit:
@@ -407,7 +411,7 @@ def query_external(
                     get_client(source.url, timeout).get,
                     url,
                     params=generate_params(limit, timeout, no_annotation, include_raw, no_cache),
-                    headers=generate_headers(access_token, clue_access_token),
+                    headers=generate_headers(obo_access_token=obo_access_token, access_token=access_token),
                     timeout=(timeout, timeout * 3),
                 )
                 rsp.raise_for_status()
@@ -493,15 +497,10 @@ def enrich(type_name: str, value: str, user: dict[str, Any]):  # noqa: C901
 
         finish_result = functools.partial(build_result, type_name, value, source)
 
-        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
-
-        # TODO: sa-clue support
-        if not obo_access_token and source.obo_target:
-            results[source.name] = finish_result(error="You must have a valid JWT to access this plugin.")
-            continue
-
-        if error:
-            results[source.name] = finish_result(error=error)
+        try:
+            access_token, obo_access_token = get_obo_access_token(source, user)
+        except AuthenticationException as err:
+            results[source.name] = finish_result(error=err.message)
             continue
 
         # check query against the max supported classification of the external system
@@ -527,8 +526,8 @@ def enrich(type_name: str, value: str, user: dict[str, Any]):  # noqa: C901
                     value=value,
                     limit=query_params.limit,
                     timeout=query_params.max_timeout,
-                    access_token=obo_access_token or access_token,
-                    clue_access_token=access_token if obo_access_token else None,
+                    access_token=access_token,
+                    obo_access_token=obo_access_token,
                     no_annotation=query_params.no_annotation,
                     include_raw=query_params.include_raw,
                     no_cache=query_params.no_cache,
@@ -562,8 +561,8 @@ def bulk_query_external(  # noqa: C901
     source: ExternalSource,
     limit: int,
     timeout: float,
-    access_token: str,
-    clue_access_token: str | None,
+    access_token: str | None,
+    obo_access_token: str | None,
     no_annotation: bool = False,
     no_cache: bool = False,
     include_raw: bool = True,
@@ -574,14 +573,16 @@ def bulk_query_external(  # noqa: C901
         execution_context.set_transaction(apm_transaction)
 
     with capture_span(bulk_query_external.__name__, span_type="greenlet"):
-        supported_types = type_service.all_supported_types(user, access_token=access_token).get(source.name, {})
+        supported_types = type_service.get_supported_types(
+            source.url, access_token=access_token, obo_access_token=obo_access_token
+        )
         bulk_result: dict[str, dict[str, QueryResult]] = {}
 
         filtered_data: list[Selector] = []
         for entry in data:
             bulk_result.setdefault(entry.type, {})
 
-            if entry.type not in supported_types:
+            if not supported_types or entry.type not in supported_types:
                 bulk_result[entry.type][entry.value] = build_result(entry.type, entry.value, source, "invalid_type")
                 continue
 
@@ -629,7 +630,7 @@ def bulk_query_external(  # noqa: C901
                     get_method=client.get,
                     params=generate_params(limit, timeout, no_annotation, include_raw, no_cache),
                     json=[entry.model_dump(exclude_none=True, exclude_unset=True) for entry in data],
-                    headers=generate_headers(access_token, clue_access_token),
+                    headers=generate_headers(obo_access_token=obo_access_token, access_token=access_token),
                     timeout=(timeout * 3, timeout * 3),
                 )
                 rsp.raise_for_status()
@@ -697,11 +698,6 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
         f"excluding sources [{','.join(excluded_sources)}]"
     )
 
-    access_token = request.headers.get("Authorization", type=str)
-    if not access_token:
-        raise AuthenticationException("Access token is required to enrich.")
-    access_token = access_token.split(" ")[1]
-
     if len(data) < 1:
         raise InvalidDataException("You must provide at least one value to lookup.")
 
@@ -744,16 +740,12 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
 
     greenlets: list[tuple[list[Selector], ExternalSource, Greenlet[Any, dict[str, dict[str, QueryResult]]]]] = []
     for source in selected_sources:
-        obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
-
-        if error:
-            logger.error("%s: %s", source.name, error)
-
-        # TODO: sa-clue support
-        if not obo_access_token and source.obo_target:
+        try:
+            access_token, obo_access_token = get_obo_access_token(source, user)
+        except AuthenticationException as err:
             for entry in data:
                 bulk_result[entry.type][entry.value][source.name] = build_result(
-                    entry.type, entry.value, source, "You must have a valid JWT to access this plugin."
+                    entry.type, entry.value, source, err.message
                 )
             continue
 
@@ -811,8 +803,8 @@ def bulk_enrich(data: list[Selector], user: dict[str, Any]):  # noqa: C901
                     source=source,
                     limit=query_params.limit,
                     timeout=query_params.max_timeout,
-                    access_token=obo_access_token or access_token,
-                    clue_access_token=access_token if obo_access_token else None,
+                    access_token=access_token,
+                    obo_access_token=obo_access_token,
                     no_annotation=query_params.no_annotation,
                     no_cache=query_params.no_cache,
                     include_raw=query_params.include_raw,

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/clue/services/type_service.py

@@ -5,15 +5,16 @@ from elasticapm.traces import capture_span
 from flask import request
 from requests import exceptions
 
+from clue.common.exceptions import AuthenticationException
 from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION, DEBUG, cache, config
 from clue.constants.env import DISABLE_CACHE
 from clue.constants.supported_types import SUPPORTED_TYPES
 from clue.helper.headers import generate_headers
+from clue.helper.obo import get_obo_access_token
 from clue.helper.plugin_requests import request_with_safe_redirects
 from clue.models.config import ExternalSource
 from clue.remote.datatypes.cache import RedisCache
-from clue.services import auth_service
 
 logger = get_logger(__file__)
 
@@ -28,10 +29,7 @@ def get_types_regular_expressions(user: dict[str, Any]):
     if access_token:
         access_token = access_token.split(" ")[1]
 
-    all_types = all_supported_types(
-        user,
-        access_token=access_token,
-    )
+    all_types = all_supported_types(user)
 
     type_detection = {}
 
@@ -47,7 +45,9 @@ def get_types_regular_expressions(user: dict[str, Any]):
 
 
 @cache.memoize(timeout=CACHE_TIMEOUT)
-def get_supported_types(source_url: str, access_token: str | None = None, obo_access_token: str | None = None):
+def get_supported_types(
+    source_url: str, access_token: str | None = None, obo_access_token: str | None = None
+) -> dict[str, str] | None:
     """Gets all supported types for the specified source.
 
     Args:
@@ -65,7 +65,7 @@ def get_supported_types(source_url: str, access_token: str | None = None, obo_ac
 
     logger.debug("Cache miss, polling plugin")
     with capture_span(f"GET {url}", span_type="http"):
-        headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
+        headers = generate_headers(obo_access_token=obo_access_token, access_token=access_token)
 
         try:
             rsp = request_with_safe_redirects(requests.get, url, headers=headers, timeout=3.0)
@@ -111,7 +111,7 @@ def get_supported_types(source_url: str, access_token: str | None = None, obo_ac
             return None
 
 
-def all_supported_types(user: dict[str, Any], access_token: str | None = None) -> dict[str, dict[str, str]]:
+def all_supported_types(user: dict[str, Any]) -> dict[str, dict[str, str]]:
     """Gets supported types by all sources.
 
     Args:
@@ -123,12 +123,13 @@ def all_supported_types(user: dict[str, Any], access_token: str | None = None) -
     all_types = {}
 
     for source in config.api.external_sources:
-        obo_access_token = None
-        if access_token:
-            obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
+        if not CLASSIFICATION.is_accessible(user["classification"], source.classification):
+            continue
 
-            if error:
-                logger.error("%s: %s", source.name, error)
+        try:
+            access_token, obo_access_token = get_obo_access_token(source, user)
+        except AuthenticationException:
+            continue
 
         supported_types = get_supported_types(source.url, access_token=access_token, obo_access_token=obo_access_token)
         if supported_types is not None:
@@ -142,11 +143,7 @@ def get_plugins_supported_types(user: dict[str, Any]) -> dict[str, list[str]]:
     configured_sources: list[ExternalSource] = getattr(config.api, "external_sources", [])
     available_types: dict[str, list[str]] = {}
 
-    access_token = request.headers.get("Authorization", type=str)
-    if access_token:
-        access_token = access_token.split(" ")[1]
-
-    all_types = all_supported_types(user, access_token=access_token)
+    all_types = all_supported_types(user)
 
     logger.info("Fetching sources for classification %s", user["classification"])
 

{clue_api-1.7.0.dev433 → clue_api-1.7.0.dev435}/pyproject.toml

@@ -141,7 +141,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.7.0.dev433"
+version = "1.7.0.dev435"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"