clue-api 1.0.0.dev7__py3-none-any.whl

clue/api/v1/registration.py

@@ -0,0 +1,109 @@
+from flask import request
+from pydantic import ValidationError
+
+from clue.api import bad_request, make_subapi_blueprint, no_content, ok
+from clue.common.logging import get_logger
+from clue.common.swagger import generate_swagger_docs
+from clue.config import config, get_redis
+from clue.models.config import ExternalSource
+from clue.remote.datatypes.set import Set
+from clue.security import api_login
+
+logger = get_logger(__file__)
+
+EXTERNAL_PLUGIN_SET = Set("plugin_set", host=get_redis())
+
+SUB_API = "registration"
+registration_api = make_subapi_blueprint(SUB_API, api_version=1)
+registration_api._doc = "Register external plugins"
+
+
+@generate_swagger_docs()
+@registration_api.route("/register/", methods=["POST"])
+@api_login()
+def register_application(**kwargs):
+    """Register the plugin given the provided data via REST API.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    API Call Examples:
+    /api/v1/registration/register/
+
+    Data Block:
+    [
+        {
+            "name": "test",
+            "classification": "TLP:CLEAR",
+            "max_classification": "TLP:CLEAR",
+            "url": "http://localhost:5008/",
+            "maintainer": "APA2B <apa2b-dl@cyber.gc.ca>",
+            "datahub_link": "http://example.com",
+            "documentation_link": "http://example.com"
+        },
+    ]
+
+    Result Example:
+    {
+        "api_response": "test",         # The response from the API
+        "api_error_message": "",         # Error message returned by the API
+        "api_warning": [],               # List of warnings from the API
+        "api_server_version": "1.0.0.dev0",  # Version of the API server
+        "api_status_code": 200           # Status code returned by the API
+    }
+
+    """
+    if not request.json:
+        return bad_request(err="No data provided")
+
+    try:
+        registration_request = ExternalSource(**request.json, built_in=False)
+    except ValidationError:
+        return bad_request(err="Request data could not be converted to an ExternalSource object")
+
+    config.api.external_sources.append(registration_request)
+    EXTERNAL_PLUGIN_SET.add(registration_request.model_dump(mode="json", exclude_none=True))
+
+    return ok(data=registration_request.name)
+
+
+@generate_swagger_docs()
+@registration_api.route("<plugin_id>", methods=["DELETE"])
+@api_login()
+def remove_application(plugin_id: str, **kwargs):
+    """Remove the given plugin from the external_sources list via REST API.
+
+    Variables:
+    name  => "test"
+
+    Optional Arguments:
+    None
+
+    API Call Examples:
+    /api/v1/registration/test
+
+    Result Example:
+    {
+        "response_status": "204 NO CONTENT"  # HTTP status code
+    }
+    """
+    source_to_remove = None
+
+    for source in config.api.external_sources:
+        if source.name == plugin_id and source.built_in is False:
+            source_to_remove = source
+            break
+
+    if (
+        source_to_remove is not None
+        and source_to_remove.model_dump(mode="json", exclude_none=True) in EXTERNAL_PLUGIN_SET.members()
+    ):
+        config.api.external_sources.remove(source_to_remove)
+        EXTERNAL_PLUGIN_SET.remove(source_to_remove.model_dump(mode="json", exclude_none=True))
+        logger.info(no_content(data=source_to_remove.name))
+        return no_content(data=source_to_remove.name)
+
+    return no_content(data=f"No plugin found with id: {plugin_id}")

clue/api/v1/static.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+
+from flask import request
+from flask_cors import CORS
+
+from clue.api import make_subapi_blueprint, not_found, ok
+from clue.common.swagger import generate_swagger_docs
+from clue.config import config
+from clue.security import api_login
+
+SUB_API = "static"
+static_api = make_subapi_blueprint(SUB_API, api_version=1)
+static_api._doc = "Fetch static documentation"
+
+CORS(static_api, origins=config.ui.cors_origins, supports_credentials=True)
+
+
+@generate_swagger_docs(responses={200: "A markdown file containing documentation"})
+@static_api.route("/docs", methods=["GET"])
+@api_login()
+def serve_documentation(**kwargs) -> dict[str, str]:
+    """Returns all documentation or filtered documentation if given a url param of a file name or a path
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    URL Link: /api/v1/static/docs?filter="howler"
+
+    {"howler-docs.md": "Markdown documentation of howler-docs.md"}
+
+    """
+    docs_filter = request.args.get("filter")
+
+    documentation_folder = Path.cwd() / "docs"
+
+    returned_files = {}
+
+    if docs_filter is None:
+        for file in documentation_folder.rglob("*"):
+            if file.is_file():
+                content = file.read_text(encoding="utf-8")
+                returned_files[file.name] = content
+    else:
+        for file in documentation_folder.rglob("*"):
+            if file.is_file() and docs_filter in file.name:
+                try:
+                    content = file.read_text(encoding="utf-8")
+                    returned_files[file.name] = content
+                except FileNotFoundError:
+                    return not_found(err="The file was not found")
+
+    return ok(returned_files)
+
+
+@generate_swagger_docs(responses={200: "A markdown file containing documentation"})
+@static_api.route("/docs/<filename>", methods=["GET"])
+@api_login()
+def serve_documentation_file(filename: str, **kwargs) -> dict[str, str]:
+    """Returns the specific file asked for within the route param
+
+    Variables:
+    filename (str): the specific file requested with an extension (i.e. *.md)
+
+    Arguments:
+    None
+
+    Result Example:
+    URL Link: /api/v1/static/docs/alfred-docs.md
+
+    {"markdown": "Markdown documentation of howler-docs.md"}
+
+    """
+    documentation_folder = Path.cwd() / "documentation"
+
+    if "." not in filename:
+        # Assume it's markdown
+        filename = filename + ".md"
+
+    returned_file = {}
+
+    for file in documentation_folder.rglob("*"):
+        if file.is_file() and file.name == filename:
+            content = file.read_text(encoding="utf-8")
+
+            returned_file["markdown"] = content
+
+    if "markdown" not in returned_file:
+        return not_found(err="The file does not exist or is typed incorrectly.")
+
+    return ok(returned_file)

clue/app.py

@@ -0,0 +1,166 @@
+import logging
+import os
+import re
+from typing import Any
+
+import elasticapm
+from authlib.integrations.flask_client import OAuth
+from elasticapm.contrib.flask import ElasticAPM
+from flasgger import Swagger
+from flask import Flask
+from flask.logging import default_handler
+from prometheus_client import make_wsgi_app
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+
+from clue.api.base import api
+from clue.api.v1 import apiv1
+from clue.api.v1.actions import actions_api
+from clue.api.v1.auth import auth_api
+from clue.api.v1.configs import configs_api
+from clue.api.v1.fetchers import fetchers_api
+from clue.api.v1.lookup import lookup_api
+from clue.api.v1.registration import registration_api
+from clue.api.v1.static import static_api
+from clue.common.logging import get_logger
+from clue.config import DEBUG, SECRET_KEY, cache, config
+from clue.cronjobs import setup_jobs as setup_cron_jobs
+from clue.error import errors
+from clue.healthz import healthz
+
+SESSION_COOKIE_SAMESITE = os.environ.get("BRL_SESSION_COOKIE_SAMESITE", None)
+HSTS_MAX_AGE = os.environ.get("BRL_HSTS_MAX_AGE", None)
+
+logger = get_logger(__file__)
+
+##########################
+# App settings
+current_directory = os.path.dirname(__file__)
+
+app = Flask("clue_api")
+# Disable strict check on trailing slashes for endpoints
+app.url_map.strict_slashes = False
+app.config["JSON_SORT_KEYS"] = False
+
+app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()})  # type: ignore[method-assign]
+
+swagger_template = {
+    "info": {
+        "title": "Clue API",
+        "description": "Clue is designed to provide analysts with pertinent insights wherever they engage with "
+        "data. It serves as a single portal for all enrichments, offering pro-active enrichment capabilities and "
+        "on-demand execution and scaling. Clue also features advanced insight visualizations and reusable UI "
+        "components to enhance the user experience and streamline analytic workflows.",
+    }
+}
+swagger = Swagger(
+    app,
+    template=swagger_template,
+    config={
+        "headers": [],
+        "static_url_path": "/api/swagger_static",
+        "specs": [
+            {
+                "endpoint": "apispec_v1",
+                "route": "/api/apispec_v1.json",
+                "rule_filter": lambda rule: True,  # all in
+                "model_filter": lambda tag: True,  # all in
+            }
+        ],
+        "specs_route": "/api/docs",
+    },
+)
+
+cache.init_app(app)
+
+app.logger.setLevel(60)  # This completely turns off the flask logger
+
+ssl_context = None
+logger.debug("Using flask secret key %s", re.sub(r"(.{6}).+(.{6})", r"\1...\2", SECRET_KEY))
+app.config.update(SESSION_COOKIE_SECURE=True, SECRET_KEY=SECRET_KEY, PREFERRED_URL_SCHEME="https")
+if SESSION_COOKIE_SAMESITE:
+    if SESSION_COOKIE_SAMESITE in ["Strict", "Lax"]:
+        app.config.update(SESSION_COOKIE_SAMESITE=SESSION_COOKIE_SAMESITE)
+    else:
+        raise ValueError("SESSION_COOKIE_SAMESITE must be set to 'Strict', 'Lax', or None")
+
+app.register_blueprint(healthz)
+app.register_blueprint(api)
+app.register_blueprint(apiv1)
+app.register_blueprint(errors)
+app.register_blueprint(auth_api)
+app.register_blueprint(actions_api)
+app.register_blueprint(configs_api)
+app.register_blueprint(fetchers_api)
+app.register_blueprint(lookup_api)
+app.register_blueprint(registration_api)
+app.register_blueprint(static_api)
+# Setup OAuth providers
+if config.auth.oauth.enabled:
+    providers = []
+    for name, provider in config.auth.oauth.providers.items():
+        raw_provider: dict[str, Any] = provider.model_dump(mode="json", exclude_none=True)
+
+        # Set provider name
+        raw_provider["name"] = name
+
+        # Remove clue specific fields from oAuth config
+        raw_provider.pop("auto_create", None)
+        raw_provider.pop("auto_sync", None)
+        raw_provider.pop("user_get", None)
+        raw_provider.pop("auto_properties", None)
+        raw_provider.pop("uid_regex", None)
+        raw_provider.pop("uid_format", None)
+        raw_provider.pop("user_groups", None)
+        raw_provider.pop("user_groups_data_field", None)
+        raw_provider.pop("user_groups_name_field", None)
+        raw_provider.pop("app_provider", None)
+
+        # Add the provider to the list of providers
+        providers.append(raw_provider)
+
+    if providers:
+        oauth = OAuth()
+        for raw_provider in providers:
+            oauth.register(**raw_provider)
+        oauth.init_app(app)
+
+if config.auth.allow_apikeys:
+    logger.debug(f"Allowing API Key use. Registered keys: {','.join(config.auth.apikeys.keys())}")
+
+# Setup logging
+app.logger.setLevel(logger.getEffectiveLevel())
+app.logger.removeHandler(default_handler)
+if logger.parent:
+    for ph in logger.parent.handlers:
+        app.logger.addHandler(ph)
+
+# Setup APMs
+if config.core.metrics.apm_server.server_url is not None:
+    app.logger.info(f"Exporting application metrics to: {config.core.metrics.apm_server.server_url}")
+    ElasticAPM(
+        app, client=elasticapm.Client(server_url=config.core.metrics.apm_server.server_url, service_name="enrichment")
+    )
+
+wlog = logging.getLogger("werkzeug")
+wlog.setLevel(logging.WARNING)
+if logger.parent:  # pragma: no cover
+    for h in logger.parent.handlers:
+        wlog.addHandler(h)
+
+# setup Cronjob
+setup_cron_jobs()
+
+
+def main():
+    """Runs the flask server"""
+    app.jinja_env.cache = {}
+    app.run(
+        host="0.0.0.0",  # noqa: S104
+        debug=DEBUG,
+        port=int(os.getenv("FLASK_RUN_PORT", os.getenv("PORT", 5000))),
+        ssl_context=ssl_context,
+    )
+
+
+if __name__ == "__main__":
+    main()

clue/cache/__init__.py

@@ -0,0 +1,129 @@
+from hashlib import sha256
+from typing import TYPE_CHECKING, Any, Literal, Self
+
+from flask import Flask
+from flask_caching import Cache as FlaskCache
+from pydantic import TypeAdapter
+
+from clue.common.logging import get_logger
+from clue.config import get_redis
+from clue.models.network import QueryEntry
+from clue.remote.datatypes.hash import ExpiringHash
+
+if TYPE_CHECKING:
+    from clue.plugin.utils import Params
+
+logger = get_logger(__file__)
+
+
+class Cache:
+    "Caching wrapped for local/redis cache"
+
+    __type: Literal["redis"] | Literal["local"]
+    __local_cache: FlaskCache | None
+    __redis_cache: ExpiringHash | None
+    __app: Flask
+
+    def __init__(
+        self: Self,
+        cache_name: str,
+        app: Flask,
+        type: Literal["redis"] | Literal["local"],
+        timeout: int = 5 * 60,  # five minute timeout
+        local_cache_options: dict[str, Any] | None = None,
+    ):
+        self.__app = app
+        self.__type = type
+
+        logger.debug("Enabling cache, type %s", self.__type)
+        if self.__type == "local":
+            self.__local_cache = FlaskCache(
+                self.__app,
+                config=(
+                    local_cache_options
+                    if local_cache_options is not None
+                    else {"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": timeout}
+                ),
+            )
+        else:
+            self.__redis_cache = ExpiringHash(cache_name, host=get_redis(), ttl=timeout)
+
+    def __generate_hash(self: Self, type_name: str, value: str, params: "Params") -> str:
+        "Generate a sha256 hash based on the selector"
+        hash_data = sha256(type_name.encode())
+        hash_data.update(value.encode())
+
+        hash_data.update(str(params.annotate).encode())
+        hash_data.update(str(params.raw).encode())
+        hash_data.update(str(params.limit).encode())
+
+        key = hash_data.hexdigest()
+
+        return key
+
+    def set(self: Self, type_name: str, value: str, params: "Params", data: list[QueryEntry]):
+        "Add the result of a given enrichment to the cache"
+        key = self.__generate_hash(type_name, value, params)
+
+        try:
+            serialized_data = TypeAdapter(list[QueryEntry]).dump_python(
+                data, mode="json", exclude_none=True, exclude_unset=True
+            )
+
+            if self.__type == "local":
+                if self.__local_cache is None:
+                    logger.warning("Local cache is None despite type being local")
+                    return
+
+                self.__local_cache.set(key, serialized_data)
+            else:
+                if self.__redis_cache is None:
+                    logger.warning("Redis cache is None despite type being redis")
+                    return
+
+                self.__redis_cache.set(key, serialized_data)
+        except Exception:
+            logger.exception("Error on retrieval")
+            return None
+
+    def get(self: Self, type_name: str, value: str, params: "Params") -> list[QueryEntry] | None:
+        "Add the result of a given enrichment to the cache"
+        key = self.__generate_hash(type_name, value, params)
+
+        try:
+            if self.__type == "local":
+                if self.__local_cache is None:
+                    return None
+
+                cached_result = self.__local_cache.get(key)
+            else:
+                if self.__redis_cache is None:
+                    return None
+
+                cached_result = self.__redis_cache.get(key)
+
+            if not cached_result:
+                return None
+
+            if not isinstance(cached_result, list):
+                cached_result = [cached_result]
+
+            return TypeAdapter(list[QueryEntry]).validate_python(cached_result)
+        except Exception:
+            logger.exception("Error on cache retrieval")
+            return None
+
+    def delete(self: Self, type_name: str, value: str, params: "Params"):
+        "Remove data associated with a key from the cache"
+        key = self.__generate_hash(type_name, value, params)
+
+        try:
+            if self.__type == "local":
+                if self.__local_cache:
+                    self.__local_cache.delete(key)
+            else:
+                if self.__redis_cache:
+                    self.__redis_cache.pop(key)
+        except Exception:
+            logger.exception("Error on cache deletion")
+            return None