clue-api 1.0.0.dev7__py3-none-any.whl

clue/gunicorn_config.py

@@ -0,0 +1,29 @@
+import multiprocessing
+from os import environ as env
+
+# Port to bind to
+bind = f":{int(env.get('PORT', 5000))}"
+
+# Number of processes to launch
+workers = int(env.get("WORKERS", multiprocessing.cpu_count()))
+
+# Number of concurrent handled connections
+threads = int(env.get("THREADS", 4))
+worker_connections = int(env.get("WORKER_CONNECTIONS", "1000"))
+
+# Recycle the process after X request randomized by the jitter
+max_requests = int(env.get("MAX_REQUESTS", "1000"))
+max_requests_jitter = int(env.get("MAX_REQUESTS_JITTER", "100"))
+
+# Connection timeouts
+graceful_timeout = int(env.get("GRACEFUL_TIMEOUT", "30"))
+timeout = int(env.get("TIMEOUT", "30"))
+
+# TLS/SSL Configuration
+certfile = env.get("CERTFILE")
+keyfile = env.get("KEYFILE")
+
+# Request Max Size Configuration
+limit_request_line = int(env.get("LIMIT_REQUEST_LINE", "4094"))
+limit_request_fields = int(env.get("LIMIT_REQUEST_FIELDS", "100"))
+limit_request_field_size = int(env.get("LIMIT_REQUEST_FIELD_SIZE", "8190"))

clue/healthz.py

@@ -0,0 +1,74 @@
+from flasgger import swag_from
+from flask import Blueprint, abort, make_response
+
+from clue.config import get_redis
+
+API_PREFIX = "/healthz"
+healthz = Blueprint("healthz", __name__, url_prefix=API_PREFIX)
+
+
+@swag_from(
+    {
+        "parameters": [],
+        "definitions": {},
+        "responses": {"200": {"description": "Liveness Probe"}},
+        "tags": ["Health"],
+        "operationId": "clue.healthz.liveness",
+    }
+)
+@healthz.route("/live")
+def liveness(**_):
+    """Check if the API is live
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Data Block:
+    None
+
+    Result example:
+    OK or FAIL
+    """
+    return make_response("OK")
+
+
+@swag_from(
+    {
+        "parameters": [],
+        "definitions": {},
+        "responses": {"200": {"description": "Readyness Probe"}},
+        "tags": ["Health"],
+        "operationId": "clue.healthz.readyness",
+    }
+)
+@healthz.route("/ready")
+def readyness(**_):
+    """Check if the API is Ready
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Data Block:
+    None
+
+    Result example:
+    OK or FAIL
+    """
+    redis = get_redis()
+
+    if redis.ping():
+        return make_response("OK")
+    else:
+        abort(503)
+
+
+@healthz.errorhandler(503)
+def error(_):
+    "Handle errors exposed in healthz routes"
+    return "FAIL", 503

clue/helper/discover.py

@@ -0,0 +1,53 @@
+import os
+import sys
+
+import geventhttpclient
+
+from clue.common.logging import get_logger
+from clue.config import config
+
+logger = get_logger(__file__)
+
+
+def get_apps_list() -> list[dict[str, str]]:
+    """Get a list of apps from the discovery service
+
+    Returns:
+        list[dict[str, str]]: A list of other apps
+    """
+    apps = []
+
+    if "pytest" in sys.modules or bool(os.getenv("SKIP_DISCOVERY", "")):
+        logger.info("Skipping discovery, running in a test environment")
+
+    if config.api.discover_url:
+        try:
+            resp = geventhttpclient.get(
+                config.api.discover_url,
+                headers={"accept": "application/json"},
+            )
+            if resp.ok:
+                data = resp.json()
+                for app in data["applications"]["application"]:
+                    try:
+                        url = app["instance"][0]["hostName"]
+
+                        if "clue" not in url:
+                            apps.append(
+                                {
+                                    "alt": app["instance"][0]["metadata"]["alternateText"],
+                                    "name": app["name"],
+                                    "img_d": app["instance"][0]["metadata"]["imageDark"],
+                                    "img_l": app["instance"][0]["metadata"]["imageLight"],
+                                    "route": url,
+                                    "classification": app["instance"][0]["metadata"]["classification"],
+                                }
+                            )
+                    except Exception:
+                        logger.exception(f"Failed to parse get app: {str(app)}")
+            else:
+                logger.warning(f"Invalid response from server for apps discovery: {config.api.discover_url}")
+        except Exception:
+            logger.exception(f"Failed to get apps from discover URL: {config.api.discover_url}")
+
+    return sorted(apps, key=lambda k: k["name"])

clue/helper/headers.py

@@ -0,0 +1,30 @@
+from clue.common.logging import get_logger
+from clue.config import DEBUG, cache, config
+
+logger = get_logger(__file__)
+
+
+@cache.memoize(timeout=1 if DEBUG else 5 * 60)  # Cached for 5 minutes
+def generate_headers(access_token: str | None, clue_access_token: str | None) -> dict[str, str]:
+    """Generates the request headers.
+
+    Args:
+        access_token (str): The access token to include in the Authorization header.
+
+    Returns:
+        dict[str, str]: A dict of the request headers
+    """
+    _headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+    }
+
+    if access_token:
+        logger.debug("Appending authorization header")
+        _headers["Authorization"] = f"Bearer {access_token}"
+
+    if config.auth.propagate_clue_key and clue_access_token:
+        logger.debug("Appending custom authorization header")
+        _headers["X-Clue-Authorization"] = clue_access_token
+
+    return _headers

clue/helper/oauth.py

@@ -0,0 +1,128 @@
+import hashlib
+import re
+from typing import Any, Optional
+
+import elasticapm
+
+from clue.common.logging import get_logger
+from clue.config import CLASSIFICATION as CL_ENGINE
+from clue.config import USER_TYPES, config
+from clue.models.config import (
+    DEFAULT_EMAIL_FIELDS,
+    DEFAULT_USER_FIELDS,
+    DEFAULT_USER_NAME_FIELDS,
+    OAuthProvider,
+)
+
+VALID_CHARS = [str(x) for x in range(10)] + [chr(x + 65) for x in range(26)] + [chr(x + 97) for x in range(26)] + ["-"]
+
+logger = get_logger(__file__)
+
+
+def reorder_name(name: Optional[str]) -> Optional[str]:
+    """Reorders a name, so that the last name goes in front of the first name.
+
+    Args:
+        name (Optional[str]): The name to reorder
+
+    Returns:
+        Optional[str]: The reordered name
+    """
+    if name is None:
+        return name
+
+    return " ".join(name.split(", ", 1)[::-1])
+
+
+@elasticapm.capture_span(span_type="authentication")
+def parse_profile(profile: dict[str, Any], provider_config: OAuthProvider) -> dict[str, Any]:  # noqa: C901
+    """Find email address and normalize it for further processing"""
+    email_adr: str | None = None
+    for field in DEFAULT_EMAIL_FIELDS:
+        if field in profile:
+            email_adr = profile[field]
+            if isinstance(email_adr, list):
+                email_adr = email_adr[0]
+            break
+
+    if isinstance(email_adr, list):
+        email_adr = email_adr[0]
+
+    if email_adr:
+        email_adr = email_adr.lower()
+        if "@" not in email_adr:
+            email_adr = None
+
+    # Find the name of the user
+    name = None
+    for field in DEFAULT_USER_NAME_FIELDS:
+        if field in profile:
+            name = reorder_name(profile[field])
+            break
+
+    # Try to find a username or use email address
+    uname = None
+    for field in DEFAULT_USER_FIELDS:
+        if field in profile:
+            uname: str = profile[field]
+            break
+    uname = uname or email_adr
+
+    # Did we use the email address?
+    if uname is not None and email_adr is not None and uname.lower() == email_adr.lower():
+        # 1. Use provided regex matcher
+        if provider_config.uid_regex:
+            match = re.match(provider_config.uid_regex, uname)
+            if match:
+                if provider_config.uid_format:
+                    uname = provider_config.uid_format.format(*[x or "" for x in match.groups()]).lower()
+                else:
+                    uname = "".join([x for x in match.groups() if x]).lower()
+
+        # 2. Parse name and domain from email if regex failed or missing
+        if uname is not None and uname == email_adr:
+            e_name, e_dom = uname.split("@", 1)
+            uname = f"{e_name}-{e_dom.split('.')[0]}"
+
+    # 3. Use name as username if there are no username found yet
+    if uname is None and name is not None:
+        uname = name.replace(" ", "-")
+
+    # Cleanup username
+    if uname:
+        uname = "".join([c for c in uname if c in VALID_CHARS])
+
+    # Get avatar from gravatar
+    if config.auth.oauth.gravatar_enabled and email_adr:
+        email_hash = hashlib.md5(email_adr.encode("utf-8")).hexdigest()  # noqa: S324
+        alternate = f"https://www.gravatar.com/avatar/{email_hash}?s=256&d=404&r=pg"
+    else:
+        alternate = None
+
+    # Compute access, roles and classification using auto_properties
+    access = True
+    roles = ["user"]
+    # TODO: correctly figure out the classification
+    classification = CL_ENGINE.UNRESTRICTED
+
+    # Infer roles from groups
+    if profile.get("groups") and provider_config.role_map:
+        for user_type in USER_TYPES:
+            if (
+                user_type in provider_config.role_map
+                and provider_config.role_map[user_type] in profile.get("groups", [])
+                and user_type not in roles
+            ):
+                roles.append(user_type)
+
+    return dict(
+        access=access,
+        type=roles,
+        classification=classification,
+        uname=uname,
+        name=name,
+        email=email_adr,
+        password="__NO_PASSWORD__",  # noqa: S106
+        avatar=profile.get("picture", alternate),
+        groups=profile.get("groups", []),
+    )

clue/models/actions.py

@@ -0,0 +1,243 @@
+# ruff: noqa: D101
+import re
+from inspect import isclass
+from typing import (
+    Any,
+    Generic,
+    Literal,
+    TypeVar,
+    Union,
+    cast,
+    get_args,
+    get_origin,
+)
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator
+from pydantic_core import Url, ValidationError
+from typing_extensions import Self
+
+from clue.common.exceptions import ClueValueError
+from clue.common.logging import get_logger
+from clue.constants.supported_types import SUPPORTED_TYPES
+from clue.models.results import DATA
+from clue.models.results.validation import validate_result
+from clue.models.selector import Selector
+from clue.models.validators import validate_classification
+
+logger = get_logger(__file__)
+
+
+class ExecuteRequest(BaseModel):
+    selector: Selector | None = Field(description="The selector to execute the action on.", default=None)
+    selectors: list[Selector] = Field(description="The selectors to execute the action on.", default=[])
+
+    @model_validator(mode="after")
+    def validate_model(self: Self, info: ValidationInfo) -> Self:  # noqa: C901
+        """Validates the entire model.
+
+        Raises:
+            AssertionError: Raised whenever a field is invalid on the model.
+
+        Returns:
+            Self: The validated model.
+        """
+        action_to_validate: Action | None = None
+        if info.context:
+            action_to_validate: Action | None = info.context.get("action", None)
+
+        if self.selector is None and (self.selectors is None or len(self.selectors) < 1):
+            if not action_to_validate or not action_to_validate.accept_empty:
+                raise ClueValueError(
+                    "Either selector (single entry) or selectors (multiple entries) must not be empty."
+                )
+        elif self.selectors is None or len(self.selectors) < 1:
+            self.selectors = [cast(Selector, self.selector)]
+        elif self.selector is None and len(self.selectors) == 1:
+            self.selector = self.selectors[0]
+
+        return self
+
+
+ER = TypeVar("ER", bound=ExecuteRequest)
+
+
+class ActionBase(BaseModel):
+    id: str = Field(description="Unique identifier for the action.")
+    name: str = Field(description="Name of the action.")
+    classification: str = Field(
+        description="Classification of the action. Denotes the maximum classification of data sent to the action.",
+    )
+    summary: str | None = Field(description="A plaintext summary of the action.", default=None)
+    supported_types: set[str] = Field(description="A list of types this action supports.")
+    action_icon: str | None = Field(
+        description=(
+            "Formatted string to present an icon for this analytic on the UI using iconify/react format: "
+            "https://iconify.design/docs/icon-components/react/. External icons not yet supported."
+        ),
+        default=None,
+    )
+    accept_empty: bool = Field(description="Does this action support execution with no selectors?", default=False)
+    accept_multiple: bool = Field(description="Does this action support multiple values?", default=False)
+    format: str | None = Field(
+        description="What is the format of the output, if known?",
+        default=None,
+    )
+    extra_schema: Any | None = Field(
+        description="Extra key values for the form schema. These will overwrite default behaviour", default={}
+    )
+
+    @field_validator("id")
+    @classmethod
+    def validate_id(cls, action_id: str) -> str:  # noqa: ANN102
+        """Validates the action ID field.
+
+        Args:
+            action_id (str): The ID to validate.
+
+        Raises:
+            ClueValueError: Raised whenever the ID is not in a valid format.
+
+        Returns:
+            str: The validated ID.
+        """
+        if re.match(r"[^a-z_]", action_id):
+            raise ClueValueError("Invalid action id - can only contain lowercase letters and underscores.")
+
+        return action_id
+
+    @field_validator("classification")
+    @classmethod
+    def check_classification(cls, classification: str) -> str:  # noqa: ANN102
+        """Validates the provided classification.
+
+        Args:
+            classification (str): The classification to validate.
+
+        Raises:
+            AssertionError: Raised whenever the provided classification is not valid.
+
+        Returns:
+            str: The validated classification.
+        """
+        return validate_classification(classification)
+
+    @field_validator("supported_types")
+    @classmethod
+    def validate_supported_types(cls, supported_types: set[str]) -> set[str]:  # noqa: ANN102
+        """Validate that the list of supported types matches the list of supported types"""
+        invalid_types = supported_types - set(SUPPORTED_TYPES.keys())
+
+        if invalid_types:
+            raise AssertionError(f"{', '.join(invalid_types)} are not supported types.")
+
+        return supported_types
+
+
+class Action(ActionBase, Generic[ER]):
+    params: ER | dict[str, Any] | None = Field(description="Specification of additional parameters.", default=None)
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_structure(cls, data: Any) -> Any:  # noqa: ANN102
+        """Checks the structure of the model.
+
+        Args:
+            data (Any): The model data to validate.
+
+        Raises:
+            ClueValueError: Raised whenever the additional_annotations field doesn't inherit ExecuteRequest
+            AssertionError: Raised whenever a field is not valid.
+
+        Returns:
+            Any: The validated data.
+        """
+        additional_annotations: type[Any] = cast(type[Any], cls.model_fields["params"].annotation).__args__[0]
+
+        if not isinstance(data.get("params", None), dict) and isinstance(additional_annotations, TypeVar):
+            raise ClueValueError(
+                "you must provide a non-generic class as a type annotation. To accept no additional parameters, use "
+                "Action[ExecuteRequest]."
+            )
+
+        if isinstance(data.get("params", None), dict):
+            if "$defs" not in data["params"]:
+                raise ClueValueError("If params is a dict, it must be a valid json schema.")
+
+            return data
+        elif not issubclass(additional_annotations, ExecuteRequest):
+            raise ClueValueError(
+                "params does not inherit from ExecuteRequest. When extending the params, it is necessary to inherit "
+                "from ExecuteRequest."
+            )
+
+        missing_annotations = [key for key, info in additional_annotations.model_fields.items() if not info.annotation]
+
+        if missing_annotations:
+            raise AssertionError(
+                f"{','.join(missing_annotations)} do not have type annotations. All fields must be annotated"
+            )
+
+        nested_fields: list[str] = []
+        for key, info in additional_annotations.model_fields.items():
+            field_type = cast(type[Any], info.annotation)
+
+            if get_origin(field_type) is Union:
+                field_type = get_args(field_type)[0]
+
+            if key not in ["selector", "selectors"] and isclass(field_type) and BaseModel in field_type.__mro__:
+                nested_fields.append(key)
+
+        if nested_fields:
+            raise AssertionError(
+                f"{','.join(nested_fields)} are not primitive types. params cannot require nested fields, "
+                "except raw_data."
+            )
+
+        return data
+
+
+class ActionResult(BaseModel, Generic[DATA]):
+    outcome: Union[Literal["success"], Literal["failure"]] = Field(description="Did the action succeed or fail?")
+    summary: str | None = Field(description="Message explaining the outcome of the action.", default=None)
+    output: DATA | Url | None = Field(description="The output of the action.", default=None)
+    format: str | None = Field(
+        description="What is the format of the output? Used to indicate what component to use when rendering "
+        "the output.",
+        default=None,
+    )
+    link: Url | None = Field(description="Link to more information on the outcome of the action", default=None)
+
+    @model_validator(mode="after")
+    def validate_model(self: Self, info: ValidationInfo) -> Self:  # noqa: C901
+        """Validates the entire model.
+
+        Raises:
+            AssertionError: Raised whenever a field is invalid on the model.
+
+        Returns:
+            Self: The validated model.
+        """
+        if not self.format and self.outcome != "failure":
+            raise ClueValueError("You must set a format if outcome is not failure.")
+
+        if self.format == "pivot" and (not self.output or not isinstance(self.output, Url)):
+            if isinstance(self.output, str):
+                try:
+                    self.output = Url(self.output)
+                    return self
+                except ValidationError:
+                    pass
+
+            raise ClueValueError("When returning a pivot, output must be a Url.")
+
+        if self.format != "pivot" and isinstance(self.output, Url):
+            raise ClueValueError("You can only return a Url if format is set to pivot.")
+
+        if self.format and not isinstance(self.output, Url):
+            self.output = validate_result(self.format, self.output, info)
+
+        return self
+
+
+class ActionSpec(ActionBase):
+    params: dict[str, Any]