howler-api 3.0.0.dev374__py3-none-any.whl

howler/services/jwt_service.py

@@ -0,0 +1,158 @@
+# implementation based on this stackoverflow post:
+# https://stackoverflow.com/a/67943659
+
+
+from typing import Any, Optional
+
+import jwt
+import requests
+from jwt.api_jwk import PyJWK
+
+from howler.common.exceptions import ForbiddenException, HowlerKeyError, HowlerValueError
+from howler.common.logging import get_logger
+from howler.config import cache, config
+
+logger = get_logger(__file__)
+
+
+def get_jwk(access_token: str) -> PyJWK:
+    """Get the JSON Web Key associated with the given JWT"""
+    # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
+    kid = jwt.get_unverified_header(access_token).get("kid")
+    jwks, _ = get_jwks()
+
+    try:
+        # Check to see if we have it cached
+        key = PyJWK(jwks[kid])
+    except KeyError:
+        # We don't, so we need to refresh the key set
+        cache.delete(key="get_jwks")
+        try:
+            jwks, _ = get_jwks()
+            key = jwks[kid]
+        except KeyError:
+            raise HowlerKeyError("Specified Key Set does not exist.")
+
+    return key
+
+
+def get_provider(access_token: str) -> str:
+    """Get the provider of a given access token
+
+    Args:
+        access_token (str): The access token to determine the provider of
+
+    Raises:
+        HowlerValueError: The provider of this access token does not match any supported providers
+
+    Returns:
+        str: The provider of the token
+    """
+    # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
+    kid = jwt.get_unverified_header(access_token).get("kid")
+    _, providers = get_jwks()
+
+    try:
+        # Check to see if we have it cached
+        oauth_provider = providers[kid]
+    except KeyError:
+        # We don't, so we need to refresh the key set
+        cache.delete(key="get_jwks")
+        try:
+            _, providers = get_jwks()
+            oauth_provider = providers[kid]
+        except KeyError:
+            raise HowlerValueError("The provider of this access token does not match any supported providers")
+
+    return oauth_provider
+
+
+@cache.cached(timeout=60 * 60 * 12, key_prefix="get_jwks")  # Cached for 12hrs
+def get_jwks() -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
+    """Get the JSON Web Key Set for all supported providers
+
+    Returns:
+        tuple[dict[str, str], dict[str, str]]: The JWKS and the providers that are included in it
+    """
+    # JWKS = JSON Web Key Set. We merge the key set from all oauth providers
+    jwks: dict[str, dict[str, Any]] = {}
+    # Mapping of keys to their provider (i.e. azure, keycloak)
+    providers: dict[str, str] = {}
+
+    for (
+        provider_name,
+        provider_data,
+    ) in config.auth.oauth.providers.items():
+        # Fetch the JSON Web Key Set for each provider that supports them
+        if provider_data.jwks_uri:
+            provider_jwks: list[dict[str, Any]] = requests.get(provider_data.jwks_uri, timeout=10).json()["keys"]
+            for jwk in provider_jwks:
+                jwks[jwk["kid"]] = jwk
+                providers[jwk["kid"]] = provider_name
+
+    return (jwks, providers)
+
+
+def get_audience(oauth_provider: str) -> str:
+    """Get the audience for the specified OAuth provider
+
+    Args:
+        oauth_provider (str): The OAuth provider to retrieve the audience of
+
+    Raises:
+        HowlerValueError: The provider is azure, and is improperly formatted
+
+    Returns:
+        str: The audience of the provider
+    """
+    audience: str = "howler"
+    provider_data = config.auth.oauth.providers[oauth_provider]
+    if provider_data.audience:
+        audience = provider_data.audience
+    elif provider_data.client_id:
+        audience = provider_data.client_id
+
+    if oauth_provider == "azure" and f"{audience}/.default" not in provider_data.scope:
+        raise HowlerValueError("Azure scope must contain the <client_id>/.default claim!")
+
+    return audience
+
+
+def decode(
+    access_token: str,
+    key: Optional[str] = None,
+    algorithms: Optional[list[str]] = None,
+    audience: Optional[str] = None,
+    validate_audience: bool = False,
+    **kwargs,
+) -> dict[str, Any]:
+    """Decode an access token into a JSON Web Token dict
+
+    Args:
+        access_token (str): The access token to decode
+        key (Optional[str], optional): The key used to sign the token. Defaults to None.
+        algorithms (Optional[list[str]], optional): The algorithm to use when decoding. Defaults to None.
+        audience (Optional[str], optional): The audience to check against, if validating the audience. Defaults to None.
+        validate_audience (bool, optional): Should we validate the audience? Defaults to False.
+
+    Returns:
+        dict[str, Any]: The decoded JWT, in dict format
+    """
+    if not key:
+        key = get_jwk(access_token).key
+
+    if not algorithms:
+        algorithms = [jwt.get_unverified_header(access_token).get("alg", "HS256")]
+
+    if validate_audience and not audience:
+        audience = get_audience(get_provider(access_token))
+
+    try:
+        logger.debug("Validating token against audience %s", audience)
+        return jwt.decode(jwt=access_token, key=key, algorithms=algorithms, audience=audience, **kwargs)  # type: ignore
+    except jwt.ExpiredSignatureError as err:
+        logger.info("JWT has expired.")
+        raise ForbiddenException("Your JWT has expired.", cause=err)
+    except jwt.InvalidTokenError as err:
+        logger.exception("Error occurred when decoding JWT.")
+        raise HowlerValueError("There was an error when decoding your JWT.", cause=err)

howler/services/lucene_service.py

@@ -0,0 +1,286 @@
+import fnmatch
+import os
+import re
+import sys
+from datetime import datetime, timedelta
+from hashlib import sha256
+from typing import Any, Literal, Union, cast
+
+from elasticsearch._sync.client.indices import IndicesClient
+from luqum.parser import parser
+from luqum.tree import AndOperation, BoolOperation, Phrase, Plus, Prohibit, Range, SearchField, Word
+from luqum.utils import UnknownOperationResolver
+from luqum.visitor import TreeVisitor
+
+from howler.api import get_logger
+from howler.common.exceptions import InvalidDataException
+from howler.common.loader import datastore
+from howler.config import redis
+from howler.remote.datatypes.hash import Hash
+from howler.utils.dict_utils import flatten_deep
+from howler.utils.lucene import coerce, normalize_phrase, try_parse_date, try_parse_ip, try_parse_number
+
+logger = get_logger(__file__)
+
+TRANSPORT_TIMEOUT = int(os.environ.get("HWL_DATASTORE_TRANSPORT_TIMEOUT", "10"))
+
+
+class LuceneProcessor(TreeVisitor):
+    "Tree visitor that evaluates a query on a given object"
+
+    def visit(self, tree: Any, context: dict[str, Any]) -> bool:
+        "Visit each node in a tree"
+        return super().visit(tree, context)[0]
+
+    def visit_search_field(self, node: SearchField, context: dict[str, Any]):
+        "Handle search fields"
+        # The actual validation happens in the word/phrases directly, not the search field.
+        # We pass the field name down for use later
+        for result in self.generic_visit(node, {**context, "field": node.name}):
+            yield result
+
+    def visit_and_operation(self, node: AndOperation, context: dict[str, Any]):
+        "Handle AND results in query"
+        yield all(list(self.generic_visit(node, context)))
+
+    def visit_or_operation(self, node: AndOperation, context: dict[str, Any]):
+        "Handle OR results in query"
+        yield any(list(self.generic_visit(node, context)))
+
+    def visit_bool_operation(self, node: BoolOperation, context: dict[str, Any]):
+        """Handle the insanity that is boolean operations.
+
+        For information about how boolean operations work, see the following extremely helpful article:
+
+            https://lucidworks.com/resources/solr-boolean-operators/
+
+        However, we are operating in a boolean environment instead of rankings, so the behaviour is slightly modified.
+        """
+        results: list[bool] = []
+        for child in node.children:
+            child_context = self.child_context(node, child, context)
+            for result in self.visit_iter(child, context=child_context):
+                # If we run across a MUST or MUST NOT (plus, probhit) object and the value doesn't match, we immediately
+                # shortcircuit and return false.
+                if isinstance(child, Plus) and not result:
+                    yield False
+                    return
+                elif isinstance(child, Prohibit) and result:
+                    yield False
+                    return
+
+                # Otherwise, we use a basic OR operation to return a result.
+                results.append(result)
+
+        yield any(results)
+
+    @staticmethod
+    def __parse_range(low: str, value: Union[list[str], str], high: str) -> Any:
+        "Generate the low, value and high components of a range check, ensuring correct types"
+        if datetime_result := coerce(value, try_parse_date):
+            low_datetime_result = cast(Any, datetime.fromtimestamp(int(low) / 1000, tz=datetime_result.tzinfo))
+
+            high_datetime_result = datetime.fromtimestamp(int(high) / 1000, tz=datetime_result.tzinfo)
+            high_datetime_result += timedelta(milliseconds=1)
+
+            return low_datetime_result, datetime_result, high_datetime_result
+
+        if number_result := coerce(value, try_parse_number):
+            low_number_result = coerce(low, try_parse_number)
+            high_number_result = coerce(high, try_parse_number)
+
+            if low_number_result is not None and high_number_result is not None:
+                return low_number_result, number_result, high_number_result
+
+        try:
+            # Check if the value is a simple integer
+            return int(low), coerce(value, int), int(high)
+        except ValueError:
+            pass
+
+        if ip_result := coerce(value, try_parse_ip):
+            low_ip_result = coerce(low, try_parse_ip)
+            high_ip_result = coerce(high, try_parse_ip)
+
+            if low_ip_result is not None and high_ip_result is not None:
+                return low_ip_result, ip_result, high_ip_result
+
+        try:
+            # Check if the value is a float
+            return float(low), coerce(value, float), float(high)
+        except ValueError:
+            pass
+
+        raise InvalidDataException(f"Unknown range type for values {low} - {value} - {high}")
+
+    def visit_range(self, node: Range, context: dict[str, Any]):
+        "Handle range queries"
+        low, value, high = self.__parse_range(node.low.value, context["hit"].get(context["field"]), node.high.value)
+
+        if isinstance(value, list):
+            values = value
+        else:
+            values = [value]
+
+        result = False
+        for _value in values:
+            if low <= _value and _value <= high:
+                if not node.include_high and _value == high:
+                    continue
+                elif not node.include_low and _value == low:
+                    continue
+
+                result = True
+                break
+
+        yield result
+
+    @staticmethod
+    def __sanitize_value(value: str) -> str:
+        "Sanitize the value we are validating against"
+        # True/False are shorthanded by elastic - convert back to True/False
+        sanitized_value = re.sub(r"^F$", r"False", value)
+        sanitized_value = re.sub(r"^T$", r"True", sanitized_value)
+
+        # For phrases, remove the encapsulating quotations
+        sanitized_value = re.sub(r'"(.+)"', r"\1", sanitized_value)
+
+        # Unescape escaped colons in value
+        sanitized_value = sanitized_value.replace("\\:", ":")
+
+        return sanitized_value
+
+    @staticmethod
+    def __build_candidates(value: Union[list[str], str], type: Union[Literal["phrase"], Literal["word"]]) -> list[str]:
+        candidates: list[str] = []
+        if isinstance(value, list):
+            for entry in value:
+                candidates += normalize_phrase(str(entry), type)
+        else:
+            candidates = normalize_phrase(str(value), type)
+
+        return candidates
+
+    def __handle_word_or_phrase(self, node: Union[Phrase, Word], context: dict[str, Any]):
+        sanitized_value = self.__sanitize_value(node.value)
+
+        if "field" not in context:
+            yield any(value == sanitized_value for value in context["hit"].values())
+        elif context["field"] == "_exists_":
+            yield context["hit"].get(node.value) is not None
+        else:
+            candidates = self.__build_candidates(context["hit"].get(context["field"]), context["term_type"])
+
+            yield len(fnmatch.filter(candidates, sanitized_value)) > 0
+
+    def visit_word(self, node: Phrase, context: dict[str, Any]):
+        "Handle words"
+        yield from self.__handle_word_or_phrase(node, {**context, "term_type": "word"})
+
+    def visit_phrase(self, node: Phrase, context: dict[str, Any]):
+        "Handle phrases"
+        yield from self.__handle_word_or_phrase(node, {**context, "term_type": "phrase"})
+
+    def visit_prohibit(self, node: Prohibit, context: dict[str, Any]):
+        "Handle NOT operation"
+        yield from (not entry for entry in self.generic_visit(node, context))
+
+
+NORMALIZED_QUERY_CACHE: Hash[str] = Hash("normalized_queries", redis)
+
+SEARCH_PHRASE_CACHE: dict[str, re.Match[str]] = {}
+
+
+def replace_lucene_phrase(match: re.Match[str]) -> str:
+    "Replace a phrase in lucene with its sha256 hash, to circumvent mangling by ES"
+    result = match.group(2) or ""
+
+    value = match.group(3)
+
+    if try_parse_date(value.replace('"', "")):
+        result += value
+    elif try_parse_ip(value.replace('"', "")):
+        result += value.replace(":", "@colon")
+    else:
+        key = sha256(value.encode()).hexdigest()
+
+        SEARCH_PHRASE_CACHE[key] = match
+
+        result += key
+
+    result += match.group(4) or ""
+
+    return result
+
+
+def try_reinsert_lucene_phrase(match: re.Match[str]) -> str:
+    "Given a potential sha256 hash, replace that hash with the original lucene phrase (if it exists)"
+    key = match.group(1)
+
+    if key in SEARCH_PHRASE_CACHE:
+        return SEARCH_PHRASE_CACHE[key].group(3)
+    else:
+        return key
+
+
+def match(lucene: str, obj: dict[str, Any]):
+    "Check if a given lucene query matches the given object"
+    hash_key = sha256(lucene.encode()).hexdigest()
+
+    # We cache the results back from ES, since we will frequently run the same validation queries over and over again.
+    if (normalized_query := NORMALIZED_QUERY_CACHE.get(hash_key)) is None or "pytest" in sys.modules:
+        # This regex checks for lucene phrases (i.e. the "Example Analytic" part of howler.analytic:"Example Analytic")
+        # And then escapes them.
+        # https://regex101.com/r/8u5F6a/1
+        escaped_lucene = re.sub(r'((:\()?(".+?")(\)?))', replace_lucene_phrase, lucene)
+
+        # This may seem unintuitive, but elastic parses lucene queries in somewhat nonstandard ways (or at least,
+        # in ways luqum doesn't agree with). to circumvent this, we use validate_query, which returns a "normalized"
+        # query that works much better with luqum. It's also much faster than actually searching for the hit in
+        # question.
+        indices_client = IndicesClient(datastore().hit.datastore.client)
+        result = indices_client.validate_query(q=escaped_lucene, explain=True, index=datastore().hit.index_name)
+
+        if not result["valid"]:
+            logger.error("Invalid lucene query:\n%s", result["explanations"][0]["error"])
+            return False
+
+        # As an example, the query:
+        #   server.address:("supports" OR "their") AND howler.votes.benign:("edge" OR "also")
+        # becomes:
+        #   +(server.address:supports server.address:their) +(howler.votes.benign:edge howler.votes.benign:also)
+        # which means the two are equivalent in elastic, but the second one is a lot less ambiguous to parse.
+        normalized_query = cast(str, result["explanations"][0]["explanation"])
+
+        # Elastic's explanation mangles exists queries. Since we will handle them the normal way, reset their changes
+        normalized_query = re.sub(r"FieldExistsQuery *\[.*?field=(.+?)]", r"_exists_:\1", normalized_query)
+        normalized_query = re.sub(r"ConstantScore", "", normalized_query)
+        # try and reinsert any phrases we have replaced with sha256 hashes
+        normalized_query = re.sub(r"([0-9a-f]{64})", try_reinsert_lucene_phrase, normalized_query)
+
+        # Properly convert escaped colons back
+        normalized_query = normalized_query.replace("@colon", ":")
+
+        # Cache the normalized query
+        NORMALIZED_QUERY_CACHE.set(hash_key, normalized_query)
+
+    try:
+        # luqum's default tree will return UnknownOperations in cases where expilicit operators aren't used.
+        # Due to the normalization step undertaken by elastic, we know that all unknown operations are actually
+        # Boolean operations.
+        #
+        # NOTE: Boolean operations have a special meaning in lucene, and are not analgous to and/or operations.
+        # For more information, see: https://lucidworks.com/resources/solr-boolean-operators/
+        tree = UnknownOperationResolver(resolve_to=BoolOperation)(parser.parse(normalized_query))
+
+        # Actually run the validation
+        return LuceneProcessor(track_parents=True).visit(tree, {"hit": flatten_deep(obj)})
+    except Exception:
+        logger.exception("Exception on processing lucene:")
+        return False
+
+
+if __name__ == "__main__":
+    hit = datastore().hit.search("howler.id:*", rows=1, as_obj=False)["items"][0]
+
+    print(match(sys.argv[1], hit))  # noqa: T201

howler/services/notebook_service.py

@@ -0,0 +1,119 @@
+from typing import Any, Callable, Optional
+
+import chevron
+import requests
+from flask import request
+
+from howler.common.exceptions import AuthenticationException, HowlerRuntimeError, HowlerValueError
+from howler.common.logging import get_logger
+from howler.config import cache, config
+from howler.odm.models.analytic import Analytic
+from howler.plugins import get_plugins
+
+logger = get_logger(__file__)
+
+
+@cache.memoize(15 * 60)
+def get_token(access_token: str) -> str:
+    """Get a notebook token based on the current howler token"""
+    get_notebook_token: Optional[Callable[[str], str]] = None
+
+    for plugin in get_plugins():
+        if get_notebook_token := plugin.modules.token_functions.get("notebook", None):
+            break
+        else:
+            logger.info("Plugin %s does not modify the notebook access token.")
+
+    if get_notebook_token:
+        notebook_access_token = get_notebook_token(access_token)
+    else:
+        logger.info("No custom notebook token logic provided, continuing with howler credentials")
+        notebook_access_token = access_token
+
+    return notebook_access_token
+
+
+def get_nbgallery_nb(link: str):
+    """Get a notebook from a given nbgallery link"""
+    # /notebooks/1-example-nb
+    # get the id (1)
+    nb_id = link.rsplit("/", 1)[-1].rsplit("-")[0]
+    auth_data: Optional[str] = request.headers.get("Authorization", None, type=str)
+
+    if not auth_data:
+        raise AuthenticationException("No Authorization header present")
+
+    access_token = get_token(auth_data.split(" ")[1])
+
+    # use obo token to retrieve notebook value
+    notebook_req = requests.get(
+        f"{config.core.notebook.url}/notebooks/{nb_id}/download.json",
+        headers={
+            "accept": "application/json",
+            "Authorization": f"Bearer {access_token}",
+        },
+        timeout=5,
+    )
+
+    if notebook_req.ok:
+        notebook: dict[str, Any] = notebook_req.json()
+
+        name = notebook["metadata"]["gallery"]["title"]
+
+        return (notebook, name)
+    else:
+        return None, None
+
+
+def get_user_envs():
+    """Get a user's environments from nbgallery"""
+    auth_data: Optional[str] = request.headers.get("Authorization", None, type=str)
+
+    if not auth_data:
+        raise AuthenticationException("No Authorization header present")
+
+    access_token = get_token(auth_data.split(" ")[1])
+
+    # get environment info from jupyterhub
+    # how to get environment without nbgallery?
+    # https://nbgallery.dev.analysis.cyber.gc.ca/environments.json
+    env = requests.get(
+        f"{config.core.notebook.url}/environments.json",
+        headers={
+            "accept": "application/json",
+            "Authorization": f"Bearer {access_token}",
+        },
+        timeout=5,
+    )
+
+    if env.ok:
+        env = env.json()
+    else:
+        raise HowlerRuntimeError(f"NBGallery returned {env.status_code}")
+
+    return env
+
+
+def get_nb_information(nb_link: str, analytic: Analytic, hit: dict[str, Any]):
+    """Get a information about a notebook from nbgallery"""
+    # get notebook
+    # only from nbgallery for now
+    if "nbgallery" in nb_link:
+        json_content, name = get_nbgallery_nb(nb_link)
+    else:
+        raise HowlerValueError("Invalid notebook source")
+
+    if not json_content or not name:
+        raise HowlerRuntimeError("An error occurred when retrieving the notebook")
+
+    try:
+        # patch first node containing code with hit/analytic info
+        cell_to_template = next(filter(lambda cell: cell["cell_type"] == "code", json_content["cells"]))
+        # goal: support any field from a hit/analytic object
+        cell_to_template["source"] = chevron.render(cell_to_template["source"], {"hit": hit, "analytic": analytic})
+    except StopIteration as e:
+        raise HowlerValueError("Notebook doesn't contain a cell with code.", e)
+    except Exception as e:
+        raise HowlerRuntimeError("Unexpected error while processing notebook.", e)
+
+    return (json_content, name)

howler/services/overview_service.py

@@ -0,0 +1,44 @@
+from typing import Any, Union
+
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.datastore.exceptions import SearchException
+from howler.odm.models.hit import Hit
+from howler.odm.models.overview import Overview
+from howler.utils.str_utils import sanitize_lucene_query
+
+logger = get_logger(__file__)
+
+
+def get_matching_overviews(
+    hits: Union[list[Hit], list[dict[str, Any]]], as_odm: bool = False
+) -> Union[list[dict[str, Any]], list[Overview]]:
+    """Generate a list of overviews matching a given list of analytic names from the provided hits.
+
+    Args:
+        hits (list[Hit] | list[dict[str, Any]]): A list of Hit objects or dictionaries containing analytic information.
+        as_odm (bool, optional): If True, return Overview objects; otherwise, return dictionaries. Defaults to False.
+
+    Returns:
+        list[dict[str, Any]] | list[Overview]: A list of matching overviews, either as dictionaries or Overview objects.
+    """
+    if len(hits) < 1:
+        return []
+
+    analytic_names: set[str] = set()
+    for hit in hits:
+        analytic_names.add(f'"{sanitize_lucene_query(hit["howler"]["analytic"])}"')
+
+    if len(analytic_names) < 1:
+        return []
+
+    try:
+        overview_candidates = datastore().overview.search(
+            f"analytic:({' OR '.join(analytic_names)})",
+            as_obj=as_odm,
+        )["items"]
+
+        return overview_candidates
+    except SearchException:
+        logger.exception("Exception on analytic matching")
+        return []

howler/services/template_service.py

@@ -0,0 +1,45 @@
+from typing import Any, Optional, Union
+
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.datastore.exceptions import SearchException
+from howler.odm.models.analytic import Analytic
+from howler.odm.models.hit import Hit
+from howler.utils.str_utils import sanitize_lucene_query
+
+logger = get_logger(__file__)
+
+
+def get_matching_templates(
+    hits: Union[list[Hit], list[dict[str, Any]]], uname: Optional[str] = None, as_odm: bool = False
+) -> Union[list[dict[str, Any]], list[Analytic]]:
+    """Generate a list of templates matching a given list of analytic names, and optionally a user.
+
+    Args:
+        hits (list[Hit] | list[dict[str, Any]]]: List of hits, each containing analytic information.
+        uname (Optional[str], optional): Username to filter templates by owner. Defaults to None.
+        as_odm (bool, optional): If True, return results as ODM objects. If False, return as dicts. Defaults to False.
+
+    Returns:
+        list[dict[str, Any]] | list[Analytic]: List of matching templates, either as dicts or Analytic ODM objects.
+    """
+    if len(hits) < 1:
+        return []
+
+    analytic_names: set[str] = set()
+    for hit in hits:
+        analytic_names.add(f'"{sanitize_lucene_query(hit["howler"]["analytic"])}"')
+
+    if len(analytic_names) < 1:
+        return []
+
+    try:
+        template_candidates = datastore().template.search(
+            f"analytic:({' OR '.join(analytic_names)}) AND (type:global OR owner:{uname or '*'})",
+            as_obj=as_odm,
+        )["items"]
+
+        return template_candidates
+    except SearchException:
+        logger.exception("Exception on analytic matching")
+        return []