clue-api 1.4.0.dev180__tar.gz → 1.4.0.dev184__tar.gz

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.4.0.dev180
+Version: 1.4.0.dev184
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/api/v1/actions.py

@@ -104,7 +104,7 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, **kwargs) ->
     task_id (str): the ID of the specific task to get the status of
 
     Arguments:
-    task_id (str): the celery task id to get the status of
+    None
 
 
     Result Example:

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/api/v1/fetchers.py

@@ -92,3 +92,41 @@ def run_fetcher(plugin_id: str, fetcher_id: str, **kwargs):
 
         logger.warning("Unknown error from fetcher %s.%s: %s", plugin_id, fetcher_id, err.message)
         return bad_gateway(err=err.message)
+
+
+@generate_swagger_docs(responses={200: "Successfully fetched status of fetcher"})
+@fetchers_api.route("/<plugin_id>/<fetcher_id>/status/<task_id>", methods=["GET"])
+@api_login()
+def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, **kwargs):
+    """Get the status or result of a fetcher
+
+    Variables:
+    plugin_id (str): the ID of the plugin who owns the action to execute
+    fetcher_id (str): the ID of the action to execute
+    task_id (str): the ID of the specific task to get the status of
+
+    Arguments:
+    None
+
+    Result Example:
+    {
+        "outcome": "success | failure", # was this execution a success or failure?
+        "format": "link", # What format is the output in?
+        "output": "http://example.com" # The output of the action. Can be any data structure.
+    }
+    """
+    try:
+        return ok(fetcher_service.get_fetcher_status(plugin_id, fetcher_id, task_id, kwargs["user"]))
+    except NotFoundException as err:
+        return not_found(err=err.message)
+    except ClueException as err:
+        if err.status_code == 400:
+            logger.warning(
+                "Bad request from fetcher %s.%s with task_id: %s: %s", plugin_id, fetcher_id, task_id, err.message
+            )
+            return bad_request(err=err.message)
+
+        logger.warning(
+            "Unknown error from fetcher %s.%s with task_id: %s: %s", plugin_id, fetcher_id, task_id, err.message
+        )
+        return bad_gateway(err=err.message)

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/models/fetchers.py

@@ -30,6 +30,7 @@ class FetcherDefinition(BaseModel):
     description: str = Field(description="A basic description of the fetcher's usage.")
     format: str = Field(description="The output format of the fetcher's result.")
     supported_types: set[str] = Field(description="A list of types this fetcher supports.")
+    async_result: bool = Field(description="Does this fetcher run asynchronously?", default=False)
     extra_data: Optional[Dict[str, JsonValue]] = Field(
         default=None, description="Extra data you want to define for a fetcher"
     )
@@ -101,14 +102,18 @@ class FetcherDefinition(BaseModel):
 
 
 class FetcherResult(BaseModel, Generic[DATA]):
-    outcome: Literal["success", "failure"] = Field(description="Did the fetcher succeed or fail?")
+    outcome: Literal["success", "failure", "pending"] = Field(
+        description="Did the fetcher succeed or fail, or is it pending?"
+    )
     data: DATA | None = Field(description="The output of the fetcher.", default=None)
     error: str | None = Field(description="If the fetcher failed, contains the relevant error message.", default=None)
-    format: str = Field(
+    format: str | None = Field(
         description="What is the format of the output? Used to indicate what component to use when rendering "
         "the output.",
+        default=None,
     )
     link: Optional[Url] = Field(description="Link to more information on the fetcher", default=None)
+    task_id: str | None = Field(description="The task id if the fetcher result is pending.", default=None)
 
     @model_validator(mode="after")
     def validate_model(self: Self, info: ValidationInfo) -> Self:  # noqa: C901
@@ -123,6 +128,9 @@ class FetcherResult(BaseModel, Generic[DATA]):
         if self.outcome == "success" and self.data is None:
             raise ClueValueError("Successful fetcher results must return data.")
 
+        if not self.task_id and self.outcome == "pending":
+            raise ClueValueError("task_id must be set if outcome is pending.")
+
         if self.outcome == "failure":
             if self.data is not None:
                 raise ClueValueError("Failed fetcher results cannot return data.")
@@ -133,7 +141,7 @@ class FetcherResult(BaseModel, Generic[DATA]):
         elif self.error:
             raise ClueValueError("Errors can only be specified if the outcome is failure.")
 
-        self.data = validate_result(self.format, self.data, info)
+        self.data = validate_result(self.format, self.data, info) if self.format else None
 
         return self
 
@@ -141,3 +149,7 @@ class FetcherResult(BaseModel, Generic[DATA]):
     def error_result(err: str) -> "FetcherResult":
         "Helper function to generate a failed fetcher result"
         return FetcherResult(outcome="failure", format="error", error=err)
+
+
+class FetcherStatusRequest(BaseModel):
+    task_id: str = Field(description="The task id to get the status for.")

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/plugin/__init__.py

@@ -36,7 +36,7 @@ from clue.models.actions import (
     ExecuteRequest,
 )
 from clue.models.config import Config
-from clue.models.fetchers import FetcherDefinition, FetcherResult
+from clue.models.fetchers import FetcherDefinition, FetcherResult, FetcherStatusRequest
 from clue.models.network import QueryEntry
 from clue.models.selector import Selector
 from clue.plugin.celery_app import celery_init_app
@@ -55,7 +55,8 @@ OVERRIDABLE_FUNCTIONS = [
     "liveness",  # Kubernetes liveness probe endpoint
     "readiness",  # Kubernetes readiness probe endpoint
     "run_action",  # Function to execute plugin actions
-    "get_status",  # Function to check the status or result of a pending action
+    "get_action_status",  # Function to check the status or result of a pending action
+    "get_fetcher_status",  # Function to check the status or result of a pending fetcher
     "run_fetcher",  # Function to execute plugin fetchers
     "setup_actions",  # Runtime action definition generation
     "validate_token",  # Custom authentication token validation
@@ -294,7 +295,13 @@ class CluePlugin:
     user parameters) that instance will be passed instead, and casting the argument will be necessary.
     """
 
-    get_status: Callable[[Action, ActionStatusRequest, str | None], ActionResult] | None
+    get_action_status: Callable[[Action, ActionStatusRequest, str | None], ActionResult] | None
+    """The function to get the status and result of running actions.
+
+    Accepts the selected action definition as well as an ActionStatusRequest instance which contains the
+    specific task id to check the status of.
+    """
+    get_fetcher_status: Callable[[FetcherDefinition, FetcherStatusRequest, str | None], FetcherResult] | None
     """The function to get the status and result of running actions.
 
     Accepts the selected action definition as well as an ActionStatusRequest instance which contains the
@@ -338,7 +345,9 @@ class CluePlugin:
         logger: logging.Logger | None = None,
         readiness: Callable[[], Response] = default_readiness,
         run_action: Callable[[Action, ExecuteRequest, str | None], ActionResult] | None = None,
-        get_status: Callable[[Action, ActionStatusRequest, str | None], ActionResult] | None = None,
+        get_action_status: Callable[[Action, ActionStatusRequest, str | None], ActionResult] | None = None,
+        get_fetcher_status: Callable[[FetcherDefinition, FetcherStatusRequest, str | None], FetcherResult]
+        | None = None,
         run_fetcher: Callable[[FetcherDefinition, Selector, str | None], FetcherResult] | None = None,
         setup_actions: Callable[[list[Action], str | None], list[Action]] | None = None,
         supported_types: set[str] | str | None = None,
@@ -439,7 +448,8 @@ class CluePlugin:
         self.enrich = enrich
         self.enable_celery = enable_celery
         self.run_action = run_action
-        self.get_status = get_status
+        self.get_action_status = get_action_status
+        self.get_fetcher_status = get_fetcher_status
         self.validate_token = validate_token
 
         self.fetchers = fetchers
@@ -714,14 +724,20 @@ class CluePlugin:
         )
         self.app.add_url_rule(
             "/actions/<action_id>/status/<task_id>",
-            self.get_action_status.__name__,
-            self.get_action_status,
+            self.action_status.__name__,
+            self.action_status,
             methods=["GET"],
         )
         self.app.add_url_rule("/fetchers/", self.get_fetchers.__name__, self.get_fetchers, methods=["GET"])
         self.app.add_url_rule(
             "/fetchers/<fetcher_id>", self.execute_fetcher.__name__, self.execute_fetcher, methods=["POST"]
         )
+        self.app.add_url_rule(
+            "/fetchers/<fetcher_id>/status/<task_id>",
+            self.fetcher_status.__name__,
+            self.fetcher_status,
+            methods=["GET"],
+        )
         self.app.add_url_rule("/types/", self.get_type_names.__name__, self.get_type_names, methods=["GET"])
         self.app.add_url_rule("/lookup/<type_name>/<value>/", self.lookup.__name__, self.lookup, methods=["GET"])
         self.app.add_url_rule("/lookup/", self.bulk_lookup.__name__, self.bulk_lookup, methods=["POST"])
@@ -1177,7 +1193,7 @@ class CluePlugin:
 
         return self.make_api_response(result)
 
-    def get_action_status(self: Self, action_id: str, task_id: str):  # noqa: C901
+    def action_status(self: Self, action_id: str, task_id: str):  # noqa: C901
         """Retrieves the status of the specified action.
 
         Args:
@@ -1192,7 +1208,7 @@ class CluePlugin:
                 {}, err="task id not provided. task id is required for this request.", status_code=400
             )
 
-        if not self.get_status:
+        if not self.get_action_status:
             return self.make_api_response(
                 {}, err=f"{self.app_name} does not support the get action status functions.", status_code=400
             )
@@ -1234,7 +1250,7 @@ class CluePlugin:
                 task_id,
             )
 
-            result = self.get_status(action_to_check, status_request, token)
+            result = self.get_action_status(action_to_check, status_request, token)
         except json.JSONDecodeError as e:
             self.logger.warning("JSON decoding error while getting status: %s", str(e))
 
@@ -1375,6 +1391,89 @@ class CluePlugin:
 
         return self.make_api_response(result, status_code=status_code)
 
+    def fetcher_status(self: Self, fetcher_id: str, task_id: str):  # noqa: C901
+        """Gets the status of a pending fetcher with the specified task_id
+
+        Args:
+            fetcher_id (str): The ID of the fetcher to get the status of
+            task_id (str): The ID of the pending task
+
+        Returns:
+            Response: A Response object with a FetcherResult as the body.
+        """
+        if not task_id:
+            return self.make_api_response(
+                {}, err="task id not provided. task id is required for this request.", status_code=400
+            )
+
+        if not self.get_fetcher_status:
+            return self.make_api_response(
+                {}, err=f"{self.app_name} does not support the get fetcher status functions.", status_code=400
+            )
+
+        if not self.fetchers:
+            return self.make_api_response({}, err=f"{self.app_name} does not support any fetchers.", status_code=400)
+
+        # Find the requested fetcher by ID
+        fetcher = next((fetcher for fetcher in self.fetchers if fetcher.id == fetcher_id), None)
+        if not fetcher:
+            return self.make_api_response({}, err=f"Fetcher {fetcher_id} does not exist", status_code=404)
+
+        token: str | None = None
+        if self.validate_token:
+            self.logger.debug("Executing plugin-provided token validator")
+
+            token, error = self.validate_token()
+
+            if error:
+                return self.make_api_response(None, f"Error on token validation: {error}", status_code=401)
+
+            self.logger.debug("Token is valid")
+        else:
+            self.logger.warning("No token validation provided. The access token will not be provided to the fetcher.")
+
+        status_code = 200
+        try:
+            status_request = FetcherStatusRequest(task_id=task_id)
+
+            self.logger.info("Getting status for fetcher '%s'", fetcher_id)
+
+            result = self.get_fetcher_status(fetcher, status_request, token)
+        except json.JSONDecodeError as e:
+            self.logger.warning("JSON decoding error during execution: %s", str(e))
+
+            status_code = 400
+            result = FetcherResult(
+                outcome="failure",
+                format="error",
+                error=f"Invalid request format. Response body must be valid JSON. Error: {str(e)}",
+            )
+        except ValidationError as err:
+            self.logger.warning("Validation error during execution: %s", str(err))
+            status_code = 400
+            result = FetcherResult(outcome="failure", format="error", error=str(err))
+        except ClueException as e:
+            self.logger.exception("ClueException during execution:")
+            status_code = 500
+            result = FetcherResult(
+                outcome="failure", format="error", error=f"Error encountered during execution: {e.message}"
+            )
+        except Exception as e:
+            self.logger.exception("%s during execution:", e.__class__.__name__)
+            status_code = 500
+            result = FetcherResult(
+                outcome="failure", format="error", error=f"An unknown error occurred during execution: {str(e)}"
+            )
+        finally:
+            self.logger.info("Fetcher completed.")
+
+        self.logger.info("Fetcher outcome: %s", result.outcome)
+
+        if result.error:
+            self.logger.info("Error Message: %s", result.error)
+
+        return self.make_api_response(result, status_code=status_code)
+
     def use(self, func: Callable):
         """Register a function to be used by the CluePlugin for specific operations.
 

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/services/action_service.py

@@ -196,7 +196,7 @@ def get_action_status(plugin_id: str, action_id: str, task_id: str, user: dict[s
     Args:
         plugin_id (str): The ID of the plugin.
         action_id (str): The ID of the action to run.
-        task_id (str): The celery task id to fetch the status for
+        task_id (str): The task id to fetch the status for
         user (dict[str, Any]): The user dict of the user running the action.
 
     Raises:

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/clue/services/fetcher_service.py

@@ -201,3 +201,71 @@ def run_fetcher(plugin_id: str, fetcher_id: str, user: dict[str, Any]) -> Fetche
         raise ClueException(
             f"Something went wrong when running fetcher from plugin '{plugin_id}': {err.__class__.__name__}."
         ) from err
+
+
+def get_fetcher_status(plugin_id: str, fetcher_id: str, task_id: str, user: dict[str, Any]) -> FetcherResult:
+    """Executes a specified fetcher.
+
+    Args:
+        plugin_id (str): The ID of the plugin.
+        fetcher_id (str): The ID of the action to run.
+        task_id (str): The task id to fetch the status for
+        user (dict[str, Any]): The user dict of the user running the action.
+
+    Raises:
+        NotFoundException: Raised whenever the plugin or the action doesn't exist.
+        ClueException: Raised whenever an error is returned by the plugin endpoint.
+
+    Returns:
+        ActionResult: The result of the action.
+    """
+    plugin = next((source for source in config.api.external_sources if source.name == plugin_id), None)
+
+    if not plugin:
+        raise NotFoundException(f"Plugin {plugin_id} does not exist.")
+
+    access_token = request.headers.get("Authorization", type=str)
+    if access_token:
+        access_token = access_token.split(" ")[1]
+
+    obo_access_token = None
+    if access_token:
+        obo_access_token, error = auth_service.check_obo(plugin, access_token, user["uname"])
+
+        if error:
+            logger.error("%s: %s", plugin.name, error)
+            raise AuthenticationException("Invalid token provided for this enrichment.")
+
+    headers = {"Accept": "application/json"}
+    if obo_access_token or access_token:
+        headers["Authorization"] = f"Bearer {obo_access_token or access_token}"
+
+    try:
+        req_url = urljoin(plugin.url, f"fetchers/{fetcher_id}/status/{task_id}")
+        logger.debug("Getting status for action %s with task_id %s for user %s", req_url, task_id, user["uname"])
+
+        response = requests.get(
+            req_url,
+            headers=headers,
+            timeout=request.args.get("max_timeout", 60.0, type=float),
+        )
+
+        result = response.json()
+
+        if not response.ok:
+            raise ClueException(
+                result["api_error_message"] or result["api_response"].get("error", ""), status_code=response.status_code
+            )
+
+        return FetcherResult.model_validate(result["api_response"], context={"is_response": True})
+    except ValidationError as err:
+        logger.exception("Invalid Request Body:")
+        raise ClueValueError(
+            "Validation error encountered on response body.",
+            status_code=400,
+        ) from err
+    except (JSONDecodeError, exceptions.ConnectionError) as err:
+        logger.exception(f"Something went wrong when getting the status of the fetcher from plugin '{plugin_id}'")
+        raise ClueException(
+            f"Something went wrong getting the status of fetcher from plugin '{plugin_id}': {err.__class__.__name__}."
+        ) from err

{clue_api-1.4.0.dev180 → clue_api-1.4.0.dev184}/pyproject.toml

@@ -139,7 +139,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.4.0.dev180"
+version = "1.4.0.dev184"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"