clue-api 1.7.0.dev465__tar.gz → 1.7.0.dev469__tar.gz

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.7.0.dev465
+Version: 1.7.0.dev469
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE
@@ -47,7 +47,7 @@ Requires-Dist: python-baseconv (>=1.2.2,<2.0.0) ; extra == "server"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; extra == "server"
 Requires-Dist: redis (>=5.0.3,<6.0.0)
 Requires-Dist: requests (>=2.32.5,<3.0.0)
-Requires-Dist: setuptools (<84.0.0)
+Requires-Dist: setuptools (<85.0.0)
 Requires-Dist: trino (>=0.336.0,<0.337.0)
 Project-URL: Documentation, https://github.com/CybercentreCanada/clue
 Project-URL: Homepage, https://github.com/CybercentreCanada/clue

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/api/v1/actions.py

@@ -8,12 +8,13 @@ List and execute actions
 
 from flask_cors import CORS
 
-from clue.api import internal_error, make_subapi_blueprint, not_found, ok, service_unavailable
+from clue.api import bad_request, internal_error, make_subapi_blueprint, not_found, ok, service_unavailable
 from clue.common.exceptions import ClueException, NotFoundException
 from clue.common.logging import get_logger
 from clue.common.swagger import generate_swagger_docs
 from clue.config import config
 from clue.models.actions import Action, ActionResult
+from clue.models.validators import validate_plugin_identifier, validate_task_id
 from clue.security import api_login
 from clue.services import action_service
 
@@ -85,10 +86,13 @@ def execute_action(plugin_id: str, action_id: str, **kwargs) -> ActionResult:
     }
     """
     try:
+        validate_plugin_identifier(action_id)
         return ok(action_service.execute_action(plugin_id, action_id, kwargs["user"]))
     except NotFoundException as err:
         return not_found(err=err.message)
     except ClueException as err:
+        if err.status_code == 400:
+            return bad_request(err=err.message)
         return internal_error(err=err.message)
 
 
@@ -116,12 +120,14 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, **kwargs) ->
     }
     """
     try:
-        if not task_id:
-            return internal_error(err="no task_id found in url. task_id is required for this request.")
+        validate_plugin_identifier(action_id)
+        validate_task_id(task_id)
         return ok(action_service.get_action_status(plugin_id, action_id, task_id, kwargs["user"]))
     except NotFoundException as err:
         return not_found(err=err.message)
     except ClueException as err:
+        if err.status_code == 400:
+            return bad_request(err=err.message)
         if err.status_code == 503:
             return service_unavailable(err=err.message)
         return internal_error(err=err.message)

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/api/v1/fetchers.py

@@ -14,6 +14,7 @@ from clue.common.logging import get_logger
 from clue.common.swagger import generate_swagger_docs
 from clue.config import config
 from clue.models.fetchers import FetcherDefinition
+from clue.models.validators import validate_plugin_identifier, validate_task_id
 from clue.security import api_login
 from clue.services import fetcher_service
 
@@ -82,6 +83,7 @@ def run_fetcher(plugin_id: str, fetcher_id: str, **kwargs):
     }
     """
     try:
+        validate_plugin_identifier(fetcher_id)
         return ok(fetcher_service.run_fetcher(plugin_id, fetcher_id, kwargs["user"]))
     except NotFoundException as err:
         return not_found(err=err.message)
@@ -116,6 +118,8 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, **kwargs):
     }
     """
     try:
+        validate_plugin_identifier(fetcher_id)
+        validate_task_id(task_id)
         return ok(fetcher_service.get_fetcher_status(plugin_id, fetcher_id, task_id, kwargs["user"]))
     except NotFoundException as err:
         return not_found(err=err.message)

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/helper/plugin_requests.py

@@ -1,14 +1,23 @@
 from collections.abc import Callable
 from time import monotonic
 from typing import Any, TypeAlias
-from urllib.parse import urljoin, urlsplit
+from urllib.parse import quote, urljoin, urlsplit
 
 from requests import Response
 from requests.exceptions import ConnectionError, Timeout
 
+from clue.common.exceptions import NotFoundException
+
 _Timeout: TypeAlias = int | float | tuple[int | float | None, int | float | None] | None
 
 
+def quote_plugin_path_segment(value: str) -> str:
+    """Encode an opaque plugin identifier, rejecting client-normalized dot segments."""
+    if value in {"", ".", ".."}:
+        raise NotFoundException("Invalid plugin identifier.", status_code=404)
+    return quote(value, safe="").replace(".", "%2E")
+
+
 def _timeout_budget(timeout: _Timeout) -> float | None:
     """Calculate the total duration represented by a requests timeout.
 

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/models/actions.py

@@ -1,5 +1,4 @@
 # ruff: noqa: D101
-import re
 from inspect import isclass
 from typing import (
     Any,
@@ -29,7 +28,7 @@ from clue.constants.supported_types import SUPPORTED_TYPES
 from clue.models.results import DATA
 from clue.models.results.validation import validate_result
 from clue.models.selector import Selector
-from clue.models.validators import validate_classification
+from clue.models.validators import validate_classification, validate_plugin_identifier
 
 logger = get_logger(__file__)
 
@@ -123,7 +122,10 @@ ER = TypeVar("ER", bound=ExecuteRequest)
 
 
 class ActionBase(BaseModel):
-    id: str = Field(description="Unique identifier for the action.")
+    id: str = Field(
+        description="Unique action identifier with optional dot-separated components using ASCII letters, digits, "
+        "underscores, or hyphens."
+    )
     name: str = Field(description="Name of the action.")
     classification: str = Field(
         description="Classification of the action. Denotes the maximum classification of data sent to the action.",
@@ -162,10 +164,7 @@ class ActionBase(BaseModel):
         Returns:
             str: The validated ID.
         """
-        if re.match(r"[^a-z_]", action_id):
-            raise ClueValueError("Invalid action id - can only contain lowercase letters and underscores.")
-
-        return action_id
+        return validate_plugin_identifier(action_id)
 
     @field_validator("classification")
     @classmethod

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/models/fetchers.py

@@ -1,5 +1,4 @@
 # ruff: noqa: D101
-import re
 from typing import Dict, Generic, Literal, Optional, Self
 
 from pydantic import (
@@ -17,13 +16,16 @@ from clue.common.logging import get_logger
 from clue.constants.supported_types import SUPPORTED_TYPES
 from clue.models.results import DATA, FORMAT_MAPPINGS_REVERSE
 from clue.models.results.validation import validate_result
-from clue.models.validators import validate_classification
+from clue.models.validators import validate_classification, validate_plugin_identifier
 
 logger = get_logger(__file__)
 
 
 class FetcherDefinition(BaseModel):
-    id: str = Field(description="An ID for the given fetcher. Structured as <plugin_id>.<fetcher_id>.")
+    id: str = Field(
+        description="Fetcher identifier, optionally plugin_id.fetcher_id; components use ASCII letters, digits, "
+        "underscores, or hyphens."
+    )
     classification: str = Field(
         description="Classification of the fetcher. Denotes the maximum classification of data sent to the fetcher.",
     )
@@ -49,10 +51,7 @@ class FetcherDefinition(BaseModel):
         Returns:
             str: The validated ID.
         """
-        if re.match(r"[^a-z_]", fetcher_id):
-            raise ClueValueError("Invalid fetcher id - can only contain lowercase letters and underscores.")
-
-        return fetcher_id
+        return validate_plugin_identifier(fetcher_id)
 
     @field_validator("classification")
     @classmethod

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/models/network.py

@@ -529,6 +529,9 @@ class QueryResult(ResultMetadata):
         Returns:
             list[QueryEntry]: The validated items field.
         """
+        if not CLASSIFICATION.enforce:
+            return items
+
         if not isinstance(info.context, dict) or "user" not in info.context:
             logger.warning("No user context given, dropping all query items")
             return []
@@ -538,6 +541,9 @@ class QueryResult(ResultMetadata):
     @classmethod
     def filter_items(cls, items: list[QueryEntry], user: Any) -> list[QueryEntry]:  # noqa: ANN102
         """Apply classification authorization independently of response schema validation."""
+        if not CLASSIFICATION.enforce:
+            return items
+
         user_classification = user.get("classification") if isinstance(user, dict) else None
         if (
             not isinstance(user_classification, str)

clue_api-1.7.0.dev469/clue/models/validators.py

@@ -0,0 +1,50 @@
+import re
+
+from clue.common.exceptions import ClueValueError
+from clue.config import CLASSIFICATION
+
+
+def validate_plugin_identifier(value: str) -> str:
+    """Allow safe ASCII components separated by dots, never dot segments."""
+    if re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*", value) is None:
+        raise ClueValueError("Invalid plugin identifier.", status_code=400)
+    return value
+
+
+MAX_TASK_ID_LENGTH = 256
+
+
+def validate_task_id(value: str) -> str:
+    """Accept any opaque plugin-issued task id that stays a single path segment once percent-encoded.
+
+    Task ids are minted by plugins (Celery UUIDs by default, but legacy/third-party backends may use other
+    formats), so only characters that cannot be safely encoded or routed are rejected. Callers must still encode
+    the value with ``quote_plugin_path_segment`` before building an upstream URL.
+    """
+    if (
+        not value
+        or len(value) > MAX_TASK_ID_LENGTH
+        or value in {".", ".."}
+        or "/" in value
+        or re.search(r"[\x00-\x1f\x7f]", value)
+    ):
+        raise ClueValueError("Invalid task id.", status_code=400)
+    return value
+
+
+def validate_classification(classification: str):
+    """Validates the provided classification.
+
+    Args:
+        classification (str): The classification to validate.
+
+    Raises:
+        AssertionError: Raised whenever the provided classification is not valid.
+
+    Returns:
+        str: The validated classification.
+    """
+    if not CLASSIFICATION.is_valid(classification):
+        raise AssertionError(f"{classification} is not a valid classification.")
+
+    return classification

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/services/action_service.py

@@ -5,17 +5,18 @@ from urllib.parse import urljoin
 import requests
 from elasticapm.traces import capture_span
 from flask import request
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 from requests import JSONDecodeError, exceptions
 
-from clue.common.exceptions import AuthenticationException, ClueException, NotFoundException
+from clue.common.exceptions import AuthenticationException, ClueException, ClueValueError, NotFoundException
 from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION, config
 from clue.helper.headers import generate_headers
 from clue.helper.obo import get_obo_access_token
-from clue.helper.plugin_requests import request_with_safe_redirects
+from clue.helper.plugin_requests import quote_plugin_path_segment, request_with_safe_redirects
 from clue.models.actions import ActionResult, ActionSpec
 from clue.models.config import ExternalSource
+from clue.models.validators import validate_plugin_identifier, validate_task_id
 
 logger = get_logger(__file__)
 
@@ -60,7 +61,15 @@ def get_supported_actions(
                     _raise_action_metadata_unavailable()
                 return {}
 
-            return TypeAdapter(dict[str, ActionSpec]).validate_python(result["api_response"])
+            actions = TypeAdapter(dict[str, ActionSpec]).validate_python(result["api_response"])
+            for identifier in actions:
+                validate_plugin_identifier(identifier)
+            return actions
+        except (ClueValueError, ValidationError) as err:
+            logger.warning("Invalid action metadata from %s", source.name)
+            if raise_on_error:
+                _raise_action_metadata_unavailable(err)
+            return {}
         except ClueException:
             raise
         except Exception as err:
@@ -141,6 +150,7 @@ def execute_action(plugin_id: str, action_id: str, user: dict[str, Any]) -> Acti
     Returns:
         ActionResult: The result of the action.
     """
+    validate_plugin_identifier(action_id)
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
     if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
@@ -164,7 +174,7 @@ def execute_action(plugin_id: str, action_id: str, user: dict[str, Any]) -> Acti
         parameters = {}
 
     try:
-        req_url = urljoin(plugin.url, f"actions/{action_id}")
+        req_url = urljoin(plugin.url, f"actions/{quote_plugin_path_segment(action_id)}")
         logger.debug("Executing action %s for user %s", req_url, user["uname"])
 
         response = request_with_safe_redirects(
@@ -205,6 +215,8 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, user: dict[s
     Returns:
         ActionResult: The result of the action.
     """
+    validate_plugin_identifier(action_id)
+    validate_task_id(task_id)
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
     if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
@@ -232,7 +244,9 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, user: dict[s
     remaining_timeout = max(timeout - (monotonic() - metadata_started), 0.001)
 
     try:
-        req_url = urljoin(plugin.url, f"actions/{action_id}/status/{task_id}")
+        req_url = urljoin(
+            plugin.url, f"actions/{quote_plugin_path_segment(action_id)}/status/{quote_plugin_path_segment(task_id)}"
+        )
         logger.debug("Getting status for action %s with task_id %s for user %s", req_url, task_id, user["uname"])
 
         response = request_with_safe_redirects(

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/clue/services/fetcher_service.py

@@ -19,10 +19,11 @@ from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION, config
 from clue.helper.headers import generate_headers
 from clue.helper.obo import get_obo_access_token
-from clue.helper.plugin_requests import request_with_safe_redirects
+from clue.helper.plugin_requests import quote_plugin_path_segment, request_with_safe_redirects
 from clue.models.config import ExternalSource
 from clue.models.fetchers import FetcherDefinition, FetcherResult
 from clue.models.selector import Selector
+from clue.models.validators import validate_plugin_identifier, validate_task_id
 
 logger = get_logger(__file__)
 
@@ -67,7 +68,15 @@ def get_supported_fetchers(
                     _raise_fetcher_metadata_unavailable()
                 return {}
 
-            return TypeAdapter(dict[str, FetcherDefinition]).validate_python(result["api_response"])
+            fetchers = TypeAdapter(dict[str, FetcherDefinition]).validate_python(result["api_response"])
+            for identifier in fetchers:
+                validate_plugin_identifier(identifier)
+            return fetchers
+        except (ClueValueError, ValidationError) as err:
+            logger.warning("Invalid fetcher metadata from %s", source.name)
+            if raise_on_error:
+                _raise_fetcher_metadata_unavailable(err)
+            return {}
         except ClueException:
             raise
         except Exception as err:
@@ -157,6 +166,7 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
     Returns:
         ActionResult: The result of the action.
     """
+    validate_plugin_identifier(fetcher_id)
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
     if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
@@ -188,7 +198,7 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
 
         response = request_with_safe_redirects(
             requests.post,
-            urljoin(plugin.url, f"fetchers/{fetcher_id}"),
+            urljoin(plugin.url, f"fetchers/{quote_plugin_path_segment(fetcher_id)}"),
             get_method=requests.get,
             json=parameters,
             headers=headers,
@@ -232,6 +242,8 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict
     Returns:
         ActionResult: The result of the action.
     """
+    validate_plugin_identifier(fetcher_id)
+    validate_task_id(task_id)
     plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
 
     if not plugin or not CLASSIFICATION.is_accessible(user["classification"], plugin.classification):
@@ -256,7 +268,9 @@ def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict
     remaining_timeout = max(timeout - (monotonic() - metadata_started), 0.001)
 
     try:
-        req_url = urljoin(plugin.url, f"fetchers/{fetcher_id}/status/{task_id}")
+        req_url = urljoin(
+            plugin.url, f"fetchers/{quote_plugin_path_segment(fetcher_id)}/status/{quote_plugin_path_segment(task_id)}"
+        )
         logger.debug("Getting status for action %s with task_id %s for user %s", req_url, task_id, user["uname"])
 
         response = request_with_safe_redirects(

{clue_api-1.7.0.dev465 → clue_api-1.7.0.dev469}/pyproject.toml

@@ -141,7 +141,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.7.0.dev465"
+version = "1.7.0.dev469"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"
@@ -186,7 +186,7 @@ beautifulsoup4 = "^4.13.3"
 imgkit = "^1.2.3"
 pillow = ">=11.1,<14.0"
 cart = "^1.2.3"
-setuptools = "<84.0.0"
+setuptools = "<85.0.0"
 requests = "^2.32.5"
 
 # Dependencies required for the main server

clue_api-1.7.0.dev465/clue/models/validators.py

@@ -1,19 +0,0 @@
-from clue.config import CLASSIFICATION
-
-
-def validate_classification(classification: str):
-    """Validates the provided classification.
-
-    Args:
-        classification (str): The classification to validate.
-
-    Raises:
-        AssertionError: Raised whenever the provided classification is not valid.
-
-    Returns:
-        str: The validated classification.
-    """
-    if not CLASSIFICATION.is_valid(classification):
-        raise AssertionError(f"{classification} is not a valid classification.")
-
-    return classification