clue-api 1.7.0.dev415__tar.gz → 1.7.0.dev420__tar.gz

{clue_api-1.7.0.dev415 → clue_api-1.7.0.dev420}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.7.0.dev415
+Version: 1.7.0.dev420
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.7.0.dev415 → clue_api-1.7.0.dev420}/clue/api/v1/auth.py

@@ -1,11 +1,10 @@
-import typing
 from typing import Any, Optional
 from urllib.parse import urlparse
 
 from authlib.integrations.base_client import OAuthError
 from flask import current_app, request
+from pydantic import ValidationError
 
-import clue.services.auth_service as auth_service
 import clue.services.user_service as user_service
 from clue.api import (
     bad_request,
@@ -26,6 +25,7 @@ from clue.common.logging import get_logger
 from clue.common.str_utils import default_string_value
 from clue.common.swagger import generate_swagger_docs
 from clue.config import config
+from clue.models.auth_user import Privilege
 from clue.security.utils import generate_random_secret
 
 logger = get_logger(__file__)
@@ -33,7 +33,7 @@ logger = get_logger(__file__)
 
 SUB_API = "auth"
 auth_api = make_subapi_blueprint(SUB_API, api_version=1)
-auth_api._doc = "Allow user to authenticate to the web server"
+auth_api._doc = "Allow user to authenticate to the web server"  # type: ignore
 
 logger = get_logger(__file__)
 
@@ -42,43 +42,43 @@ logger = get_logger(__file__)
 @generate_swagger_docs()
 @auth_api.route("/login", methods=["GET", "POST"])
 def login(**_) -> dict[str, Any]:  # noqa: C901
-    """Log the user into the system, in one of three ways.
-
-    1. Username/Password Authentication
-    2. Username/API Key Authentication
-    3. OAuth Login flow
-        (See here: https://auth0.com/docs/get-started/authentication-and-authorization-flow/authorization-code-flow)
-
-    Variables:
-    None
-
-    Arguments:
-    NOTE: The arguments are used only when completing the OAuth authorization flow.
-    provider    => The provider of the OAuth code.
-    state       => Random state used in the OAuth authentication flow.
-    code        => The code provided by the OAuth provider used to exchange for an access token.
-
-    Data Block:
-    {
-        "user": "user",                 # The username to authenticate as (optional)
-        "password": "password",         # The password used to authenticate (optional)
-        "apikey": "devkey:user",        # The apikey used ot authenticate (optional)
-        "oauth_provider": "keycloak"    # The oauth provider initiate an OAuth Authorization Flow with (optional)
-    }
-
-    Result Example:
-    {
-        # Profile picture for the user
-        "avatar": "data:image/png;base64, ...",
-        # Username of the authenticated user
-        "username": "user",
-        # Different privileges that the user will get for this session
-        "privileges": ["R", "W"],
-        # A token generated by us the user can use to authenticate with clue
-        "app_token": "asdfsd876opqwm465a89sdf4",
-        # A JSON Web Access Token generated by an OAuth provider to authenticate with them
-        "access_token": "<JWT>",
-    }
+    """Authenticate a user through an OAuth provider.
+
+    The initial request redirects the user to the selected OAuth provider.
+    The callback exchanges the authorization code, validates the returned user,
+    and returns the provider-issued access token as the application token.
+
+    Args:
+        provider: The OAuth provider to use for authentication.
+        oauth_provider: Alternative name for ``provider``.
+        code: The authorization code returned by the OAuth provider.
+        refresh_token: An OAuth refresh token used to obtain a new access token.
+        nonce: Optional nonce used during the OAuth authorization flow.
+
+    Returns:
+        A Clue API response containing the access token, refresh token, user
+        information, and granted privileges. An initial authentication request
+        returns the provider's authorization redirect response.
+
+    Example:
+        A successful authentication returns a response containing the provider
+        token, privileges, and normalized user details::
+
+            {
+                "app_token": "<OAuth access token>",
+                "provider": "keycloak",
+                "refresh_token": "<OAuth refresh token>",
+                "privileges": ["R", "W"],
+                "user": {
+                    "uname": "user",
+                    "name": "User Name",
+                    "email": "user@example.com",
+                    "classification": "UNCLASSIFIED",
+                    "groups": ["clue_user"],
+                    "roles": ["user"],
+                    "avatar": "data:image/png;base64, ..."
+                }
+            }
     """
     data: dict[str, Any]
     if request.is_json and len(request.data) > 0:
@@ -100,7 +100,7 @@ def login(**_) -> dict[str, Any]:  # noqa: C901
     logged_in_uname = None
     access_token = None
     refresh_token = data.get("refresh_token", None)
-    priv: Optional[list[str]] = []
+    priv: Optional[list[Privilege]] = []
 
     try:
         # First, we'll try oauth
@@ -173,12 +173,18 @@ def login(**_) -> dict[str, Any]:  # noqa: C901
             access_token = token_data.get("access_token", None)
             refresh_token = token_data.get("refresh_token", None)
 
+            if not access_token:
+                raise AuthenticationException("The OAuth provider did not return an access token.")
+
             # Get a useful dict of user data from the web token
-            cur_user = user_service.parse_user_data(token_data, oauth_provider)
+            try:
+                cur_user = user_service.parse_user_data(token_data, oauth_provider)
+            except ValidationError as e:
+                raise AuthenticationException("The OAuth provider returned invalid user information.") from e
 
-            logged_in_uname = cur_user["uname"]
+            logged_in_uname = cur_user.uname
 
-            priv = ["R", "W"]
+            priv = [Privilege.READ, Privilege.WRITE]
 
         # No oauth provider was specified, so we fall back to user/pass or user/apikey
         # elif user and (password or apikey):
@@ -226,18 +232,13 @@ def login(**_) -> dict[str, Any]:  # noqa: C901
 
     # Generate the token this user can use to authenticate from now on
 
-    if access_token:
-        app_token = access_token
-    else:
-        app_token = f"{logged_in_uname}:{auth_service.create_token(logged_in_uname, typing.cast(list[str], priv))}"
-
     return ok(
         {
-            "app_token": app_token,
+            "app_token": access_token,
             "provider": oauth_provider,
             "refresh_token": refresh_token,
             "privileges": priv,
-            "user": cur_user,
+            "user": cur_user.model_dump(mode="json"),
         },
         cookies={"XSRF-TOKEN": xsrf_token},
     )

{clue_api-1.7.0.dev415 → clue_api-1.7.0.dev420}/clue/api/v1/registration.py

@@ -5,6 +5,7 @@ from clue.api import bad_request, make_subapi_blueprint, no_content, ok
 from clue.common.logging import get_logger
 from clue.common.swagger import generate_swagger_docs
 from clue.config import config, get_redis
+from clue.models.auth_user import Privilege, UserRole
 from clue.models.config import ExternalSource
 from clue.remote.datatypes.set import Set
 from clue.security import api_login
@@ -15,12 +16,12 @@ EXTERNAL_PLUGIN_SET = Set("plugin_set", host=get_redis())
 
 SUB_API = "registration"
 registration_api = make_subapi_blueprint(SUB_API, api_version=1)
-registration_api._doc = "Register external plugins"
+registration_api._doc = "Register external plugins"  # type: ignore
 
 
 @generate_swagger_docs()
 @registration_api.route("/register/", methods=["POST"])
-@api_login()
+@api_login(required_priv=[Privilege.WRITE], required_roles=[UserRole.ADMIN])
 def register_application(**kwargs):
     """Register the plugin given the provided data via REST API.
 
@@ -56,23 +57,43 @@ def register_application(**kwargs):
     }
 
     """
-    if not request.json:
+    if not request.is_json:
+        return bad_request(err="Invalid data type.")
+
+    body = request.json
+    if not (body and isinstance(body, dict)):
         return bad_request(err="No data provided")
 
+    existing_source_names = {source.name for source in config.api.external_sources}
+    existing_source_names.update(
+        source["name"]
+        for source in EXTERNAL_PLUGIN_SET.members()
+        if isinstance(source, dict) and isinstance(source.get("name"), str)
+    )
+
     try:
-        registration_request = ExternalSource(**request.json, built_in=False)
-    except ValidationError:
-        return bad_request(err="Request data could not be converted to an ExternalSource object")
+        registration_request = ExternalSource.model_validate(
+            {**request.json, "built_in": False},
+            context={
+                "registration_allowed_origins": config.api.registration_allowed_origins,
+                "existing_source_names": existing_source_names,
+            },
+        )
+    except ValidationError as error:
+        return bad_request(err="; ".join(item["msg"] for item in error.errors()))
+
+    persisted_source = registration_request.model_dump(mode="json", exclude_none=True)
+    if not EXTERNAL_PLUGIN_SET.add_if_field_absent(persisted_source, "name", registration_request.name):
+        return bad_request(err="An external source with that name already exists")
 
     config.api.external_sources.append(registration_request)
-    EXTERNAL_PLUGIN_SET.add(registration_request.model_dump(mode="json", exclude_none=True))
 
     return ok(data=registration_request.name)
 
 
 @generate_swagger_docs()
-@registration_api.route("<plugin_id>", methods=["DELETE"])
-@api_login()
+@registration_api.route("/<plugin_id>", methods=["DELETE"])
+@api_login(required_priv=[Privilege.WRITE], required_roles=[UserRole.ADMIN])
 def remove_application(plugin_id: str, **kwargs):
     """Remove the given plugin from the external_sources list via REST API.
 
@@ -90,20 +111,18 @@ def remove_application(plugin_id: str, **kwargs):
         "response_status": "204 NO CONTENT"  # HTTP status code
     }
     """
-    source_to_remove = None
-
-    for source in config.api.external_sources:
-        if source.name == plugin_id and source.built_in is False:
-            source_to_remove = source
-            break
-
-    if (
-        source_to_remove is not None
-        and source_to_remove.model_dump(mode="json", exclude_none=True) in EXTERNAL_PLUGIN_SET.members()
-    ):
-        config.api.external_sources.remove(source_to_remove)
-        EXTERNAL_PLUGIN_SET.remove(source_to_remove.model_dump(mode="json", exclude_none=True))
-        logger.info(no_content(data=source_to_remove.name))
-        return no_content(data=source_to_remove.name)
+    persisted_sources = [
+        source
+        for source in EXTERNAL_PLUGIN_SET.members()
+        if isinstance(source, dict) and source.get("name") == plugin_id
+    ]
+
+    if persisted_sources:
+        config.api.external_sources = [
+            source for source in config.api.external_sources if source.name != plugin_id or source.built_in is True
+        ]
+        EXTERNAL_PLUGIN_SET.remove(*persisted_sources)
+        logger.info("Removed plugin with id %s", plugin_id)
+        return no_content(data=plugin_id)
 
     return no_content(data=f"No plugin found with id: {plugin_id}")

clue_api-1.7.0.dev420/clue/cronjobs/plugins.py

@@ -0,0 +1,58 @@
+from apscheduler.schedulers.base import BaseScheduler
+from gevent.queue import Queue
+from pydantic import ValidationError
+
+from clue.api.v1.registration import EXTERNAL_PLUGIN_SET
+from clue.common.logging import get_logger
+from clue.config import config
+from clue.models.config import ExternalSource
+
+logger = get_logger(__file__)
+
+config_updates = Queue()
+
+__scheduler_instance: BaseScheduler | None = None
+
+
+def update_external_source_list():
+    """Updates the external_sources list with the plugins that have been registered through the API."""
+    built_in_sources = [item for item in config.api.external_sources if item.built_in is True]
+    source_names = {source.name for source in built_in_sources}
+    plugin_list: list[ExternalSource] = []
+
+    for item in EXTERNAL_PLUGIN_SET.members():
+        if not isinstance(item, dict):
+            logger.warning("Ignoring invalid runtime external source configuration: expected an object")
+            continue
+
+        try:
+            source = ExternalSource.model_validate(
+                {**item, "built_in": False},
+                context={
+                    "registration_allowed_origins": config.api.registration_allowed_origins,
+                    "existing_source_names": source_names,
+                },
+            )
+        except ValidationError as error:
+            logger.warning(
+                "Ignoring invalid runtime external source configuration: %s",
+                "; ".join(item["msg"] for item in error.errors()),
+            )
+            continue
+
+        source_names.add(source.name)
+        plugin_list.append(source)
+
+    config.api.external_sources = built_in_sources + plugin_list
+
+
+def setup_job(sched: BaseScheduler):
+    """Sets the scheduler instance to the one provided, and refreshes the external sources.
+
+    Arguments:
+        sched: The scheduler instance to set.
+    """
+    global __scheduler_instance
+    __scheduler_instance = sched
+    sched.add_job(update_external_source_list, "interval", minutes=1)
+    logger.debug("Plugin job setup complete.")

{clue_api-1.7.0.dev415 → clue_api-1.7.0.dev420}/clue/helper/oauth.py

@@ -6,7 +6,8 @@ import elasticapm
 
 from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION as CL_ENGINE
-from clue.config import USER_TYPES, config
+from clue.config import config
+from clue.models.auth_user import UserRole
 from clue.models.config import (
     DEFAULT_EMAIL_FIELDS,
     DEFAULT_USER_FIELDS,
@@ -34,7 +35,7 @@ def reorder_name(name: Optional[str]) -> Optional[str]:
     return " ".join(name.split(", ", 1)[::-1])
 
 
-@elasticapm.capture_span(span_type="authentication")
+@elasticapm.capture_span(span_type="authentication")  # type: ignore
 def parse_profile(profile: dict[str, Any], provider_config: OAuthProvider) -> dict[str, Any]:  # noqa: C901
     """Find email address and normalize it for further processing"""
     email_adr: str | None = None
@@ -61,10 +62,10 @@ def parse_profile(profile: dict[str, Any], provider_config: OAuthProvider) -> di
             break
 
     # Try to find a username or use email address
-    uname = None
+    uname: str | None = None
     for field in DEFAULT_USER_FIELDS:
         if field in profile:
-            uname: str = profile[field]
+            uname = profile[field]
             break
     uname = uname or email_adr
 
@@ -101,28 +102,27 @@ def parse_profile(profile: dict[str, Any], provider_config: OAuthProvider) -> di
 
     # Compute access, roles and classification using auto_properties
     access = True
-    roles = ["user"]
+    roles = [UserRole.USER]
     # TODO: correctly figure out the classification
     classification = CL_ENGINE.UNRESTRICTED
 
     # Infer roles from groups
     if profile.get("groups") and provider_config.role_map:
-        for user_type in USER_TYPES:
+        for role in UserRole:
             if (
-                user_type in provider_config.role_map
-                and provider_config.role_map[user_type] in profile.get("groups", [])
-                and user_type not in roles
+                role in provider_config.role_map
+                and provider_config.role_map[role] in profile.get("groups", [])
+                and role not in roles
             ):
-                roles.append(user_type)
+                roles.append(role)
 
     return dict(
         access=access,
-        type=roles,
+        roles=roles,
         classification=classification,
         uname=uname,
         name=name,
         email=email_adr,
-        password="__NO_PASSWORD__",  # noqa: S106
         avatar=profile.get("picture", alternate),
         groups=profile.get("groups", []),
     )

clue_api-1.7.0.dev420/clue/helper/plugin_requests.py

@@ -0,0 +1,234 @@
+from collections.abc import Callable
+from time import monotonic
+from typing import Any, TypeAlias
+from urllib.parse import urljoin, urlsplit
+
+from requests import Response
+from requests.exceptions import ConnectionError, Timeout
+
+_Timeout: TypeAlias = int | float | tuple[int | float | None, int | float | None] | None
+
+
+def _timeout_budget(timeout: _Timeout) -> float | None:
+    """Calculate the total duration represented by a requests timeout.
+
+    Requests accepts either a scalar timeout or a (connect, read) pair. A
+    timeout with an unbounded component has no finite total budget.
+
+    Args:
+        timeout: A numeric timeout, a connect/read timeout pair, or ``None``.
+
+    Returns:
+        The total timeout in seconds, or ``None`` when no finite budget exists.
+    """
+    if isinstance(timeout, tuple):
+        # If either phase is unbounded, do not pretend the pair has a deadline.
+        connect_timeout, read_timeout = timeout
+        if connect_timeout is None or read_timeout is None:
+            return None
+        return float(connect_timeout) + float(read_timeout) or None
+    return float(timeout) if timeout else None
+
+
+def _remaining_timeout(timeout: _Timeout, remaining: float) -> _Timeout:
+    """Fit an original timeout into the remaining overall request budget.
+
+    Args:
+        timeout: The original scalar or connect/read timeout setting.
+        remaining: Time left before the overall deadline, in seconds.
+
+    Returns:
+        A timeout value that does not exceed the remaining budget. An
+        unbounded setting is preserved as-is.
+    """
+    if isinstance(timeout, tuple):
+        budget = _timeout_budget(timeout)
+        if budget is None:
+            return timeout
+        # Preserve the connect/read ratio while shrinking both phases together.
+        scale = min(1.0, remaining / budget)
+        connect_timeout, read_timeout = timeout
+        if connect_timeout is None or read_timeout is None:
+            return timeout
+        return float(connect_timeout) * scale, float(read_timeout) * scale
+    if timeout is None:
+        return None
+    return min(float(timeout), remaining)
+
+
+def _switch_to_get(status_code: int, get_method: Callable[..., Response] | None) -> bool:
+    """Return whether a redirect should use the supplied GET method.
+
+    Args:
+        status_code: HTTP status code returned by the redirect response.
+        get_method: Optional GET callable to use for method-changing redirects.
+
+    Returns:
+        Whether a GET method is available and the status commonly changes the
+        redirected request to GET.
+    """
+    return get_method is not None and status_code in {301, 302, 303}
+
+
+def _clear_request_body(kwargs: dict[str, Any]) -> None:
+    """Remove payload and query arguments from a redirected GET request.
+
+    Args:
+        kwargs: Request keyword arguments, updated in place.
+    """
+    for key in ("json", "data", "files", "params"):
+        kwargs.pop(key, None)
+
+
+def _next_redirect(
+    response: Response,
+    url: str,
+    request_method: Callable[..., Response],
+    get_method: Callable[..., Response] | None,
+    kwargs: dict[str, Any],
+    redirect_count: int,
+) -> tuple[Callable[..., Response], str]:
+    """Validate and prepare the next hop in a redirect chain.
+
+    Args:
+        response: The response that requested a redirect.
+        url: The URL used for the current request.
+        request_method: The method callable currently used for requests.
+        get_method: Optional callable used when a redirect changes to GET.
+        kwargs: Request keyword arguments, updated for the next hop.
+        redirect_count: Number of redirects already followed.
+
+    Returns:
+        The request callable and absolute URL to use for the next hop. When a
+        response has no Location header, the current method and URL are returned.
+
+    Raises:
+        ConnectionError: If the redirect target is invalid, unsafe, or exceeds
+            the redirect limit.
+    """
+    location = response.headers.get("Location")
+    if not location:
+        return request_method, url
+
+    try:
+        # Resolve relative locations against the response URL, as HTTP requires.
+        target_url = urljoin(response.url or url, location)
+    except ValueError as err:
+        response.close()
+        raise ConnectionError("Plugin redirected to an untrusted URL") from err
+    if not _safe_redirect(url, target_url):
+        response.close()
+        raise ConnectionError("Plugin redirected to an untrusted URL")
+    if redirect_count == 5:
+        response.close()
+        raise ConnectionError("Plugin exceeded the redirect limit")
+
+    # Some redirect codes turn a request into GET; others retain its method.
+    switch_to_get = _switch_to_get(response.status_code, get_method)
+    if switch_to_get:
+        request_method = get_method  # type: ignore[assignment]
+    if response.status_code == 303 or switch_to_get:
+        _clear_request_body(kwargs)
+    else:
+        kwargs.pop("params", None)
+
+    response.close()
+    return request_method, target_url
+
+
+def _safe_redirect(current_url: str, target_url: str) -> bool:
+    """Check whether a redirect stays on the same trusted host and port.
+
+    Redirects are permitted within the same origin, and from the default HTTP
+    origin to the default HTTPS origin on the same host. Other host, port,
+    scheme, and embedded-credential changes are rejected.
+
+    Args:
+        current_url: URL from which the redirect originated.
+        target_url: Resolved URL requested by the redirect response.
+
+    Returns:
+        Whether the redirect target satisfies the allowed origin rules.
+    """
+    try:
+        current = urlsplit(current_url)
+        target = urlsplit(target_url)
+        # Reject non-web schemes, malformed hosts, and credential-bearing URLs.
+        if (
+            target.scheme not in {"http", "https"}
+            or not target.hostname
+            or target.username is not None
+            or target.password is not None
+        ):
+            return False
+        current_port = current.port or (443 if current.scheme == "https" else 80)
+        target_port = target.port or (443 if target.scheme == "https" else 80)
+    except ValueError:
+        return False
+
+    # Never let a plugin redirect send request headers to a different host.
+    if current.hostname != target.hostname:
+        return False
+    # Permit same-origin redirects or only the conventional HTTP-to-HTTPS upgrade.
+    if current.scheme == target.scheme and current_port == target_port:
+        return True
+    return current.scheme == "http" and current_port == 80 and target.scheme == "https" and target_port == 443
+
+
+def request_with_safe_redirects(
+    request_method: Callable[..., Response],
+    url: str,
+    *,
+    get_method: Callable[..., Response] | None = None,
+    **kwargs: Any,
+) -> Response:
+    """Send a request while safely following a bounded chain of redirects.
+
+    Redirects are handled manually so each destination can be checked before
+    sending the request again. A single total timeout budget is maintained
+    across all hops when a finite timeout is supplied.
+
+    Args:
+        request_method: Callable used to send the initial request and redirects
+            that preserve the method.
+        url: Initial request URL.
+        get_method: Optional callable to use when a redirect changes the method
+            to GET.
+        **kwargs: Additional keyword arguments passed to the request callable.
+            ``allow_redirects`` is always disabled so this function can validate
+            each redirect itself.
+
+    Returns:
+        The first response that does not request a redirect.
+
+    Raises:
+        ConnectionError: If a redirect is unsafe or the redirect limit is
+            exceeded.
+        Timeout: If the total timeout budget expires between redirect hops.
+    """
+    # Disable requests' automatic redirects to validate every target ourselves.
+    kwargs["allow_redirects"] = False
+    timeout = kwargs.get("timeout")
+    budget = _timeout_budget(timeout)
+    # Reuse one deadline so a chain cannot receive a fresh timeout per hop.
+    deadline = monotonic() + budget if budget is not None else None
+
+    for redirect_count in range(6):
+        if deadline is not None and redirect_count > 0:
+            remaining = deadline - monotonic()
+            if remaining <= 0:
+                raise Timeout("Plugin request exceeded its total timeout")
+            # Pass only the time left to the next network operation.
+            kwargs["timeout"] = _remaining_timeout(timeout, remaining)
+
+        response = request_method(url, **kwargs)
+        if response.status_code not in {301, 302, 303, 307, 308}:
+            return response
+
+        next_method, next_url = _next_redirect(response, url, request_method, get_method, kwargs, redirect_count)
+        # A redirect status without Location is not actionable; return it intact.
+        if next_url == url and not response.headers.get("Location"):
+            return response
+        request_method, url = next_method, next_url
+
+    raise ConnectionError("Plugin exceeded the redirect limit")