clue-api 1.0.0.dev7__py3-none-any.whl

clue/.gitignore

@@ -0,0 +1,21 @@
+# Add any directories, files, or patterns you don't want to be tracked by version control
+
+# IDE files
+.pydevproject
+.python-version
+.idea
+
+# Testing artifacts
+.pytest_cache
+htmlcov
+.coverage
+
+# OS Files
+ehthumbs.db
+Thumbs.db
+
+# Build artifacts
+build/
+dist/
+*.py[cod]
+*.egg-info

clue/api/__init__.py

@@ -0,0 +1,211 @@
+from sys import exc_info
+from traceback import format_tb
+from typing import Any, Union
+
+from flask import Blueprint, Response, make_response, request
+from prometheus_client import Counter
+
+from clue.common.forge import APP_NAME
+from clue.common.logging import get_logger, log_with_traceback
+from clue.common.str_utils import safe_str
+from clue.models.network import ClueResponse
+
+API_PREFIX = "/api"
+RAW_API_COUNTER = Counter(
+    f"{APP_NAME.replace('-', '_')}_http_requests_total",  # type: ignore[union-attr]
+    "HTTP Requests broken down by method, path, and status",
+    ["method", "path", "status"],
+)
+
+logger = get_logger(__file__)
+
+
+def make_subapi_blueprint(name, api_version=1):
+    """Create a flask Blueprint for a subapi in a standard way."""
+    return Blueprint(name, name, url_prefix="/".join([API_PREFIX, f"v{api_version}", name]))
+
+
+def _make_api_response(
+    data: Any, err: Union[str, Exception] = "", warnings: list[str] = [], status_code: int = 200, cookies: Any = None
+) -> Response:
+    if isinstance(err, Exception):  # pragma: no cover
+        trace = exc_info()[2]
+        err = "".join(["\n"] + format_tb(trace) + ["%s: %s\n" % (err.__class__.__name__, str(err))]).rstrip("\n")
+        log_with_traceback(trace, "Exception", is_exception=True)
+
+    resp = make_response(
+        ClueResponse(response=data, error_message=err, warning=warnings, status_code=status_code).model_dump(
+            mode="json", by_alias=True, exclude_none=True
+        ),
+        status_code,
+    )
+
+    resp.headers["Content-Type"] = "application/json"
+
+    if isinstance(cookies, dict):
+        for k, v in cookies.items():
+            resp.set_cookie(k, v)
+
+    RAW_API_COUNTER.labels(request.method, str(request.url_rule), status_code).inc()
+    logger.info("%s %s - %s", request.method, request.path, status_code)
+
+    return resp
+
+
+# Some helper functions for make_api_response
+
+DEFAULT_DATA = {True: {"success": True}, False: {"success": False}}
+
+
+def ok(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 200"""
+    return _make_api_response(data, status_code=200, cookies=cookies)
+
+
+def created(data=DEFAULT_DATA[True], warnings=[], cookies=None):
+    """Returns response with status code 201"""
+    return _make_api_response(data, warnings=warnings, status_code=201, cookies=cookies)
+
+
+def accepted(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 202"""
+    return _make_api_response(data, status_code=202, cookies=cookies)
+
+
+def no_content(data=None, cookies=None):
+    """Returns response with status code 204"""
+    return _make_api_response(data or DEFAULT_DATA[True], status_code=204, cookies=cookies)
+
+
+def not_modified(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 304"""
+    return _make_api_response(data, status_code=304, cookies=cookies)
+
+
+def bad_request(data=DEFAULT_DATA[False], err="", cookies=None, warnings=[]):
+    """Returns response with status code ies"""
+    return _make_api_response(data, err, status_code=400, cookies=cookies, warnings=warnings)
+
+
+def unauthorized(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 401"""
+    return _make_api_response(data, err, status_code=401, cookies=cookies)
+
+
+def forbidden(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 403"""
+    return _make_api_response(data, err, status_code=403, cookies=cookies)
+
+
+def not_found(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 404"""
+    return _make_api_response(data, err, status_code=404, cookies=cookies)
+
+
+def conflict(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 409"""
+    return _make_api_response(data, err, status_code=409, cookies=cookies)
+
+
+def precondition_failed(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 412"""
+    return _make_api_response(data, err, status_code=412, cookies=cookies)
+
+
+def teapot(data={**DEFAULT_DATA[False], "teapot": True}, err="", cookies=None):
+    """Returns response with status code 418"""
+    return _make_api_response(data, err, status_code=418, cookies=cookies)
+
+
+def too_many_requests(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 429"""
+    return _make_api_response(data, err, status_code=429, cookies=cookies)
+
+
+def internal_error(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 500"""
+    return _make_api_response(data, err, status_code=500, cookies=cookies)
+
+
+def not_implemented(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 501"""
+    return _make_api_response(data, err, status_code=501, cookies=cookies)
+
+
+def bad_gateway(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 502"""
+    return _make_api_response(data, err, status_code=502, cookies=cookies)
+
+
+def service_unavailable(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 503"""
+    return _make_api_response(data, err, status_code=503, cookies=cookies)
+
+
+def make_file_response(data, name, size, status_code=200, content_type="application/octet-stream"):
+    """Returns file response with arbitrary status code"""
+    response = make_response(data, status_code)
+    response.headers["Content-Type"] = content_type
+    response.headers["Content-Length"] = size
+    response.headers["Content-Disposition"] = 'attachment; filename="%s"' % safe_str(name)
+    return response
+
+
+def stream_file_response(reader, name, size, status_code=200):
+    """Returns stream response with arbitrary status code"""
+    chunk_size = 65535
+
+    def generate():
+        reader.seek(0)
+        while True:
+            data = reader.read(chunk_size)
+            if not data:
+                break
+            yield data
+        reader.close()
+
+    headers = {
+        "Content-Type": "application/octet-stream",
+        "Content-Length": size,
+        "Content-Disposition": 'attachment; filename="%s"' % safe_str(name),
+    }
+    return Response(generate(), status=status_code, headers=headers)
+
+
+def make_binary_response(data, size, status_code=200):
+    """Returns binary response with arbitrary status code"""
+    response = make_response(data, status_code)
+    response.headers["Content-Type"] = "application/octet-stream"
+    response.headers["Content-Length"] = size
+    return response
+
+
+def stream_binary_response(reader, status_code=200):
+    """Returns streamed binary response with arbitrary status code"""
+    chunk_size = 4096
+
+    def generate():
+        reader.seek(0)
+        while True:
+            data = reader.read(chunk_size)
+            if not data:
+                break
+            yield data
+
+    return Response(generate(), status=status_code, mimetype="application/octet-stream")

clue/api/base.py

@@ -0,0 +1,99 @@
+from flask import Blueprint, current_app, request
+
+from clue.api import ok
+from clue.common.logging import get_logger
+from clue.security import api_login
+
+logger = get_logger(__file__)
+
+API_PREFIX = "/api"
+api = Blueprint("api", __name__, url_prefix=API_PREFIX)
+
+XSRF_ENABLED = True
+
+
+#####################################
+# API list API (API inception)
+@api.route("/")
+@api_login(audit=False)
+def api_version_list(**_):
+    """List all available API versions.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Data Block:
+    None
+
+    Result example:
+    ["v1", "v2", "v3"]         #List of API versions available
+    """
+    api_list = []
+    for rule in current_app.url_map.iter_rules():
+        if rule.rule.startswith("/api/"):
+            version = rule.rule[5:].split("/", 1)[0]
+            if version not in api_list and version != "":
+                # noinspection PyBroadException
+                try:
+                    int(version[1:])
+                except ValueError:
+                    continue
+                api_list.append(version)
+
+    return ok(api_list)
+
+
+@api.route("/site_map/")
+@api_login(audit=False)
+def site_map(**_):
+    """Check if all pages have been protected by a login decorator
+
+    Variables:
+    None
+
+    Arguments:
+    unsafe_only                    => Only show unsafe pages
+
+    Data Block:
+    None
+
+    Result example:
+    [                                #List of pages dictionary containing...
+     {"function": views.default,     #Function name
+      "url": "/",                    #Url to page
+      "protected": true,             #Is function login protected
+      "methods": ["GET"]},           #Methods allowed to access the page
+    ]
+    """
+    pages = []
+    for rule in current_app.url_map.iter_rules():
+        func = current_app.view_functions[rule.endpoint]
+        methods = []
+        if rule.methods:
+            for item in rule.methods:
+                if item != "OPTIONS" and item != "HEAD":
+                    methods.append(item)
+        protected = func.__dict__.get("protected", False)
+        audit = func.__dict__.get("audit", False)
+        if "/api/v1/" in rule.rule:
+            prefix = "api.v1."
+        else:
+            prefix = ""
+
+        if "unsafe_only" in request.args and protected:
+            continue
+
+        pages.append(
+            {
+                "function": f"{prefix}{rule.endpoint.replace('apiv1.', '')}",
+                "url": rule.rule,
+                "methods": methods,
+                "protected": protected,
+                "audit": audit,
+            }
+        )
+
+    return ok(sorted(pages, key=lambda i: i["url"]))

clue/api/v1/__init__.py

@@ -0,0 +1,82 @@
+from textwrap import dedent
+
+from flask import Blueprint, current_app, request
+
+from clue.api import ok
+from clue.api.base import api_login
+
+API_PREFIX = "/api/v1"
+apiv1 = Blueprint("apiv1", __name__, url_prefix=API_PREFIX)
+apiv1._doc = "Api Documentation - Verison 1"  # type: ignore[attr-defined]
+
+
+#####################################
+# API DOCUMENTATION
+# noinspection PyProtectedMember,PyBroadException
+@apiv1.route("/")
+@api_login(audit=False)
+def get_api_documentation(**_):
+    """Full API doc. Loop through all registered API paths and display their documentation.
+
+    Returns a list of API definition.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    [
+     {'name': "Api Doc",                # Name of the api
+      'path': "/api/path/<variable>/",  # API path
+      'methods': ["GET", "POST"],       # Allowed HTTP methods
+      'description': "API doc.",        # API documentation
+      'id': "api_doc",                  # Unique ID for the API
+      'function': "apiv1.api_doc",      # Function called in the code
+      'protected': False,               # Does the API require login?
+      'complete' : True},               # Is the API stable?
+      ...]
+    """
+    api_blueprints = {}
+    api_list = []
+    for rule in current_app.url_map.iter_rules():
+        if rule.rule.startswith(request.path):
+            methods = [item for item in (rule.methods or []) if item != "OPTIONS" and item != "HEAD"]
+
+            func = current_app.view_functions[rule.endpoint]
+            doc_string = func.__doc__
+            func_title = " ".join([x.capitalize() for x in rule.endpoint[rule.endpoint.rindex(".") + 1 :].split("_")])
+            blueprint = rule.endpoint[: rule.endpoint.rindex(".")]
+            if blueprint == "apiv1":
+                blueprint = "documentation"
+
+            if blueprint not in api_blueprints:
+                try:
+                    doc = current_app.blueprints[rule.endpoint[: rule.endpoint.rindex(".")]]._doc  # type: ignore[attr-defined]
+                except Exception:
+                    doc = ""
+
+                api_blueprints[blueprint] = doc
+
+            if doc_string:
+                description = dedent(doc_string)
+            else:
+                description = "[INCOMPLETE]\n\nTHIS API HAS NOT BEEN DOCUMENTED YET!"
+
+            api_id = rule.endpoint.replace("apiv1.", "").replace(".", "_")
+
+            api_list.append(
+                {
+                    "protected": func.__dict__.get("protected", False),
+                    "name": func_title,
+                    "id": api_id,
+                    "function": f"api.v1.{rule.endpoint}",
+                    "path": rule.rule,
+                    "methods": methods,
+                    "description": description,
+                    "complete": "[INCOMPLETE]" not in description,
+                }
+            )
+
+    return ok({"apis": api_list, "blueprints": api_blueprints})

clue/api/v1/actions.py

@@ -0,0 +1,92 @@
+"""Enrichment Actions
+
+List and execute actions
+
+* Provide endpoints to list valid actions exposed by plugins.
+* Provide endpoints to execute these actions.
+"""
+
+from flask_cors import CORS
+
+from clue.api import internal_error, make_subapi_blueprint, not_found, ok
+from clue.common.exceptions import ClueException, NotFoundException
+from clue.common.logging import get_logger
+from clue.common.swagger import generate_swagger_docs
+from clue.config import config
+from clue.models.actions import Action, ActionResult
+from clue.security import api_login
+from clue.services import action_service
+
+logger = get_logger(__file__)
+
+
+SUB_API = "actions"
+actions_api = make_subapi_blueprint(SUB_API, api_version=1)
+actions_api._doc = "Run actions on data through configured external data sources/systems."
+
+CORS(actions_api, origins=config.ui.cors_origins, supports_credentials=True)
+
+
+@generate_swagger_docs(responses={200: "A list of types and their classification"})
+@actions_api.route("/", methods=["GET"])
+@api_login()
+def get_actions(**kwargs) -> dict[str, Action]:
+    """Return the supported actions of each external service.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    { # A dictionary of sources with their supported actions.
+        <source_id>.<action_id>: {
+            "id": "",
+            "name": "",
+            "classification": "",
+            "summary": "",
+            "supported_types": "",
+            "params": {
+                <JSON schema>
+            }
+        },
+        ...,
+    }
+    """
+    return ok(action_service.get_plugins_supported_actions(kwargs["user"]))
+
+
+@generate_swagger_docs(responses={200: "Successful lookup to selected plugins"})
+@actions_api.route("/execute/<plugin_id>/<action_id>", methods=["POST"])
+@api_login()
+def execute_action(plugin_id: str, action_id: str, **kwargs) -> ActionResult:
+    """Search other services for additional information related to the provided data.
+
+    Variables:
+    plugin_id (str): the ID of the plugin who owns the action to execute
+    action_id (str): the ID of the action to execute
+
+    Arguments:
+    None
+
+    Data Block:
+    {
+        type: "ip",
+        value: "127.0.0.1",
+        ...
+    }
+
+    Result Example:
+    {
+        "outcome": "success | failure", # was this execution a success or failure?
+        "format": "link", # What format is the output in?
+        "output": "http://example.com" # The output of the action. Can be any data structure.
+    }
+    """
+    try:
+        return ok(action_service.execute_action(plugin_id, action_id, kwargs["user"]))
+    except NotFoundException as err:
+        return not_found(err=err.message)
+    except ClueException as err:
+        return internal_error(err=err.message)