clue-api 1.0.0.dev7__py3-none-any.whl

clue/common/classification.yml

@@ -0,0 +1,130 @@
+# This the default classification engine provided with Clue,
+#  it showcases all the different features of the classification engine
+#  while providing a useful configuration
+
+# Turn on/off classification enforcement. When this flag is off, this
+#  completely disables the classification engine, any documents added while
+#  the classification engine is off gets the default unrestricted value
+enforce: false
+
+# Turn on/off dynamic group creation. This feature allow you to dynamically create classification groups based on
+#  features from the user.
+dynamic_groups: false
+
+# Set the type of dynamic groups to be used
+#  email: groups will be based of the user's email domain
+#  group: groups will be created out the the user's group values
+#  all: groups will be created out of both the email domain and the group values
+dynamic_groups_type: email
+
+# List of Classification level:
+#   Graded list were a smaller number is less restricted then an higher number.
+levels:
+  # List of alternate names for the current marking
+  - aliases:
+      - UNRESTRICTED
+      - UNCLASSIFIED
+      - U
+      - TLP:W
+      - TLP:WHITE
+    # Stylesheet applied in the UI for the different levels
+    css:
+      # Name of the color scheme used for display (default, primary, secondary, success, info, warning, error)
+      color: default
+    # Description of the classification level
+    description: Subject to standard copyright rules, TLP:CLEAR information may be distributed without restriction.
+    # Interger value of the Classification level (higher is more classified)
+    lvl: 100
+    # Long name of the classification item
+    name: TLP:CLEAR
+    # Short name of the classification item
+    short_name: TLP:C
+  - aliases: []
+    css:
+      color: success
+    description:
+      Recipients may share TLP:GREEN information with peers and partner organizations
+      within their sector or community, but not via publicly accessible channels. Information
+      in this category can be circulated widely within a particular community. TLP:GREEN
+      information may not be released outside of the community.
+    lvl: 110
+    name: TLP:GREEN
+    short_name: TLP:G
+  - aliases: []
+    css:
+      color: warning
+    description:
+      Recipients may only share TLP:AMBER information with members of their
+      own organization and with clients or customers who need to know the information
+      to protect themselves or prevent further harm.
+    lvl: 120
+    name: TLP:AMBER
+    short_name: TLP:A
+  - aliases:
+      - RESTRICTED
+    css:
+      color: warning
+    description:
+      Recipients may only share TLP:AMBER+STRICT information with members of their
+      own organization.
+    lvl: 125
+    name: TLP:AMBER+STRICT
+    short_name: TLP:A+S
+
+# List of required tokens:
+#   A user requesting access to an item must have all the
+#   required tokens the item has to gain access to it
+required:
+  - aliases: []
+    description: Produced using a commercial tool with limited distribution
+    name: COMMERCIAL
+    short_name: CMR
+    # The minimum classification level an item must have
+    #   for this token to be valid. (optional)
+    # require_lvl: 100
+    # This is a token that is required but will display in the groups part
+    #   of the classification string. (optional)
+    # is_required_group: true
+
+# List of groups:
+#   A user requesting access to an item must be part of a least
+#   of one the group the item is part of to gain access
+groups:
+  - aliases: []
+    # This is a special flag that when set to true, if any groups are selected
+    #   in a classification. This group will automatically be selected too. (optional)
+    auto_select: true
+    description: Employees of CSE
+    name: CSE
+    short_name: CSE
+    # Assuming that this groups is the only group selected, this is the display name
+    #   that will be used in the classification (that values has to be in the aliases
+    #   of this group and only this group) (optional)
+    # solitary_display_name: ANY
+
+# List of subgroups:
+#   A user requesting access to an item must be part of a least
+#   of one the subgroup the item is part of to gain access
+subgroups:
+  - aliases: []
+    description: Member of Incident Response team
+    name: IR TEAM
+    short_name: IR
+  - aliases: []
+    description: Member of the Canadian Centre for Cyber Security
+    # This is a special flag that auto-select the corresponding group
+    #   when this subgroup is selected (optional)
+    require_group: CSE
+    name: CCCS
+    short_name: CCCS
+    # This is a special flag that makes sure that none other then the
+    #   corresponding group is selected when this subgroup is selected (optional)
+    # limited_to_group: CSE
+
+# Default restricted classification
+restricted: TLP:A+S//CMR
+
+# Default unrestricted classification:
+#   When no classification are provided or that the classification engine is
+#   disabled, this is the classification value each items will get
+unrestricted: TLP:C

clue/common/dict_utils.py

@@ -0,0 +1,130 @@
+from collections.abc import Mapping
+from typing import Any, AnyStr, Optional, cast
+from typing import Mapping as _Mapping
+
+
+def strip_nulls(d: Any):
+    """Remove null values from a dict"""
+    if isinstance(d, dict):
+        return {k: strip_nulls(v) for k, v in d.items() if v is not None}
+    else:
+        return d
+
+
+def recursive_update(
+    d: Optional[dict[str, Any]],
+    u: Optional[_Mapping[str, Any]],
+    stop_keys: list[AnyStr] = [],
+    allow_recursion: bool = True,
+) -> dict[str, Any]:
+    "Recursively update a dict with another value"
+    if d is None:
+        return cast(dict, u or {})
+
+    if u is None:
+        return d
+
+    for k, v in u.items():
+        if isinstance(v, Mapping) and allow_recursion:
+            d[k] = recursive_update(d.get(k, {}), v, stop_keys=stop_keys, allow_recursion=k not in stop_keys)
+        else:
+            d[k] = v
+
+    return d
+
+
+def get_recursive_delta(
+    d1: Optional[_Mapping[str, Any]],
+    d2: Optional[_Mapping[str, Any]],
+    stop_keys: list[AnyStr] = [],
+    allow_recursion: bool = True,
+) -> Optional[dict[str, Any]]:
+    "Get the recursive difference between two objects"
+    if d1 is None:
+        return cast(dict, d2)
+
+    if d2 is None:
+        return cast(dict, d1)
+
+    out = {}
+    for k1, v1 in d1.items():
+        if isinstance(v1, Mapping) and allow_recursion:
+            internal = get_recursive_delta(
+                v1,
+                d2.get(k1, {}),
+                stop_keys=stop_keys,
+                allow_recursion=k1 not in stop_keys,
+            )
+            if internal:
+                out[k1] = internal
+        else:
+            if k1 in d2:
+                v2 = d2[k1]
+                if v1 != v2:
+                    out[k1] = v2
+
+    for k2, v2 in d2.items():
+        if k2 not in d1:
+            out[k2] = v2
+
+    return out
+
+
+def flatten(data: _Mapping, parent_key: Optional[str] = None) -> dict[str, Any]:
+    "Flatten a nested dict"
+    items: list[tuple[str, Any]] = []
+    for k, v in data.items():
+        cur_key = f"{parent_key}.{k}" if parent_key is not None else k
+        if isinstance(v, dict):
+            items.extend(flatten(v, cur_key).items())
+        else:
+            items.append((cur_key, v))
+
+    return dict(items)
+
+
+def unflatten(data: _Mapping) -> _Mapping:
+    "Unflatted a nested dict"
+    out: dict[str, Any] = dict()
+    for k, v in data.items():
+        parts = k.split(".")
+        d = out
+        for p in parts[:-1]:
+            if p not in d:
+                d[p] = dict()
+            d = d[p]
+        d[parts[-1]] = v
+    return out
+
+
+def prune(data: _Mapping, keys: list[str], parent_key: Optional[str] = None) -> dict[str, Any]:
+    "Remove all keys in the given list from the dict if they exist"
+    pruned_items: list[tuple[str, Any]] = []
+
+    for key, val in data.items():
+        cur_key = f"{parent_key}.{key}" if parent_key else key
+
+        if isinstance(val, dict):
+            child_keys = [_key for _key in keys if _key.startswith(cur_key)]
+
+            if len(child_keys) > 0:
+                pruned_items.append((key, prune(val, child_keys, cur_key)))
+        elif isinstance(val, list):
+            if cur_key not in keys and not any(_key.startswith(cur_key) for _key in keys):
+                continue
+
+            list_result = []
+            for entry in val:
+                if isinstance(val, dict):
+                    child_keys = [_key for _key in keys if _key.startswith(cur_key)]
+
+                    if len(child_keys) > 0:
+                        pruned_items.append((key, prune(val, child_keys, cur_key)))
+                else:
+                    list_result.append(entry)
+
+            pruned_items.append((key, list_result))
+        elif cur_key in keys:
+            pruned_items.append((key, val))
+
+    return {k: v for k, v in pruned_items}

clue/common/exceptions.py

@@ -0,0 +1,199 @@
+from inspect import getmembers, isfunction
+from sys import exc_info
+from traceback import format_tb
+from typing import Optional, Self
+
+
+class ClueException(Exception):
+    """Wrapper for all exceptions thrown in Clue' code"""
+
+    message: str
+    cause: Exception | None
+    status_code: int | None
+
+    def __init__(
+        self: Self,
+        message: str = "Something went wrong",
+        cause: Exception | None = None,
+        status_code: int | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.message = message
+        self.cause = cause
+        self.status_code = status_code
+
+    def __repr__(self: Self) -> str:
+        """String reproduction of the Clue exception. Pass the message on"""
+        return self.message
+
+
+class InvalidClassification(ClueException):
+    """Exception for Invalid Classification"""
+
+
+class InvalidDefinition(ClueException):
+    """Exception for Invalid Definition"""
+
+
+class InvalidRangeException(ClueException):
+    """Exception for Invalid Range"""
+
+
+class NonRecoverableError(ClueException):
+    """Exception for an unrecoverable error"""
+
+
+class RecoverableError(ClueException):
+    """Exception for a recoverable error"""
+
+
+class ConfigException(ClueException):
+    """Exception thrown due to invalid configuration"""
+
+
+class ResourceExists(ClueException):
+    """Exception thrown due to a pre-existing resource"""
+
+
+class VersionConflict(ClueException):
+    """Exception thrown due to a version conflict"""
+
+    def __init__(self: Self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        ClueException.__init__(self, message, cause)
+
+
+class ClueTypeError(ClueException, TypeError):
+    """TypeError child specifically for exceptions thrown by us"""
+
+    def __init__(
+        self: Self,
+        message: str = "Something went wrong",
+        cause: Optional[Exception] = None,
+        status_code: int | None = None,
+    ) -> None:
+        ClueException.__init__(
+            self, message, cause if cause is not None else TypeError(message), status_code=status_code
+        )
+
+
+class ClueAttributeError(ClueException, AttributeError):
+    """AttributeError child specifically for exceptions thrown by us"""
+
+    def __init__(
+        self: Self,
+        message: str = "Something went wrong",
+        cause: Optional[Exception] = None,
+        status_code: int | None = None,
+    ) -> None:
+        ClueException.__init__(
+            self, message, cause if cause is not None else AttributeError(message), status_code=status_code
+        )
+
+
+class ClueValueError(ClueException, ValueError):
+    """ValueError child specifically for exceptions thrown by us"""
+
+    def __init__(
+        self: Self,
+        message: str = "Something went wrong",
+        cause: Optional[Exception] = None,
+        status_code: int | None = None,
+    ) -> None:
+        ClueException.__init__(
+            self, message, cause if cause is not None else ValueError(message), status_code=status_code
+        )
+
+
+class ClueNotImplementedError(ClueException, NotImplementedError):
+    """NotImplementedError child specifically for exceptions thrown by us"""
+
+    def __init__(self: Self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        ClueException.__init__(self, message, cause if cause is not None else NotImplementedError(message))
+
+
+class ClueKeyError(ClueException, KeyError):
+    """KeyError child specifically for exceptions thrown by us"""
+
+    def __init__(self: Self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        ClueException.__init__(self, message, cause if cause is not None else KeyError(message))
+
+
+class ClueRuntimeError(ClueException, RuntimeError):
+    """RuntimeError child specifically for exceptions thrown by us"""
+
+    def __init__(self: Self, message: str = "Something went wrong", cause: Optional[Exception] = None) -> None:
+        ClueException.__init__(self, message, cause if cause is not None else RuntimeError(message))
+
+
+class NotFoundException(ClueException):
+    """Exception thrown when a resource cannot be found"""
+
+
+class AccessDeniedException(ClueException):
+    """Exception thrown when a resource cannot be accessed by a user"""
+
+
+class InvalidDataException(ClueException):
+    """Exception thrown when user-provided data is invalid"""
+
+
+class AuthenticationException(ClueException):
+    """Exception thrown when a user cannot be authenticated"""
+
+
+class TimeoutException(ClueException):
+    """Exception for Timeout"""
+
+
+class UnprocessableException(ClueException):
+    """Exception for Unprocessable"""
+
+
+class Chain(object):
+    """This class can be used as a decorator to override the type of exceptions returned by a function"""
+
+    def __init__(self: Self, exception: type[Exception]):
+        self.exception = exception
+
+    def __call__(self, original):
+        """Execute a function and wrap any resulting exceptions"""
+
+        def wrapper(*args, **kwargs):
+            try:
+                return original(*args, **kwargs)
+            except Exception as e:
+                wrapped = self.exception(str(e), e)
+                raise wrapped.with_traceback(exc_info()[2])
+
+        wrapper.__name__ = original.__name__
+        wrapper.__doc__ = original.__doc__
+        wrapper.__dict__.update(original.__dict__)
+
+        return wrapper
+
+    def execute(self, func, *args, **kwargs):
+        """Execute a function and wrap any resulting exceptions"""
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            wrapped = self.exception(str(e), e)
+            raise wrapped.with_traceback(exc_info()[2])
+
+
+class ChainAll:
+    """This class can be used as a decorator to override the type of exceptions returned by every method of a class"""
+
+    def __init__(self: Self, exception: type[Exception]):
+        self.exception = Chain(exception)
+
+    def __call__(self, cls):
+        """We can use an instance of this class as a decorator."""
+        for method in getmembers(cls, predicate=isfunction):
+            setattr(cls, method[0], self.exception(method[1]))
+
+        return cls
+
+
+def get_stacktrace_info(ex: Exception) -> str:
+    """Get and format traceback information from a given exception"""
+    return "".join(format_tb(exc_info()[2]) + [": ".join((ex.__class__.__name__, str(ex)))])

clue/common/forge.py

@@ -0,0 +1,152 @@
+# This file contains the loaders for the different components of the system
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from string import Template
+from typing import TYPE_CHECKING
+
+from flask_caching import Cache
+
+from clue.common.dict_utils import recursive_update
+from clue.common.logging.format import BRL_DATE_FORMAT, BRL_LOG_FORMAT
+from clue.common.str_utils import default_string_value
+
+APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue")  # type: ignore[assignment]
+APP_PREFIX = os.environ.get("APP_PREFIX", "brl")
+
+if TYPE_CHECKING:
+    from clue.common.classification import Classification
+
+cache = Cache(config={"CACHE_TYPE": "SimpleCache"})
+
+classification_engines: dict[Path, Classification] = {}
+
+logger = logging.getLogger(f"{APP_NAME}.common.forge")
+logger.setLevel(logging.INFO)
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
+logger.addHandler(console)
+
+
+def __get_yml_path(yml_config: str | None = None) -> Path | None:  # noqa: C901
+    if yml_config is not None:
+        return Path(yml_config)
+
+    if (_yml_path := Path(f"/etc/{APP_NAME}/classification.yml")).exists():
+        return _yml_path
+
+    if (_yml_path := Path(f"/etc/{APP_NAME}/conf/classification.yml")).exists():
+        return _yml_path
+
+    if os.getenv("AZURE_TEST_CONFIG", None) is not None:
+        import re
+
+        logger.info("Azure build environment detected, checking additional classification path")
+
+        work_dir_parent = Path("/__w")
+        work_dir: Path | None = None
+        for sub_path in work_dir_parent.iterdir():
+            if not sub_path.is_dir():
+                continue
+
+            logger.info("Testing sub path %s", sub_path)
+
+            if re.match(r"\d+", sub_path.name):
+                work_dir = work_dir_parent / sub_path
+
+            if work_dir is not None:
+                logger.info("Subpath %s exists, checking for test path", work_dir)
+                test_classification_path = work_dir / "s" / "test" / "config" / "classification.yml"
+
+                if test_classification_path.exists():
+                    logger.info("Path %s detected", test_classification_path)
+                    return test_classification_path
+
+                logger.error("No classification path found at path %s", test_classification_path)
+                logger.info(
+                    "Available files:\n%s", "\n".join(sorted(str(path) for path in (work_dir / "s").glob("**/*")))
+                )
+                work_dir = None
+
+    custom_path = os.environ.get("CLUE_CONF_FOLDER", None)
+    if custom_path is None:
+        return None
+
+    if (_yml_path := (Path(custom_path) / "classification.yml")).exists():
+        return _yml_path
+
+    return None
+
+
+def get_classification(yml_config: str | None = None):  # noqa: C901
+    """Creates and registers a Classification engine.
+
+    If a yaml config is not provided, it will search in /etc/clue and /etc/clue/conf for a classification.yml
+    file instead.
+
+    Arguments:
+        yml_config: An optional yaml config to load.
+
+    Returns:
+        The created Classification engine.
+    """
+    import yaml
+
+    from clue.common.classification import Classification, InvalidDefinition
+
+    _yml_path = __get_yml_path(yml_config)
+
+    if _yml_path:
+        logger.debug("Classification file found at %s", _yml_path)
+    else:
+        logger.warning("Missing classification.yml file!")
+
+    if _yml_path in classification_engines:
+        return classification_engines[_yml_path]
+
+    classification_definition = {}
+    default_file = Path(__file__).parent / "classification.yml"
+    if default_file.exists():
+        with default_file.open() as default_fh:
+            default_yml_data = yaml.safe_load(default_fh.read())
+            if default_yml_data:
+                classification_definition.update(default_yml_data)
+
+    # Load modifiers from the yaml config
+    if _yml_path is not None and _yml_path.exists():
+        with _yml_path.open() as yml_fh:
+            yml_data = yaml.safe_load(yml_fh.read())
+            if yml_data:
+                classification_definition = recursive_update(classification_definition, yml_data)
+
+    if not classification_definition:
+        raise InvalidDefinition("Could not find any classification definition to load.")
+
+    classification_engine = Classification(classification_definition)
+
+    if _yml_path:
+        classification_engines[_yml_path] = classification_engine
+
+    return classification_engine
+
+
+def env_substitute(buffer):
+    """Replace environment variables in the buffer with their value.
+
+    Use the built in template expansion tool that expands environment variable style strings ${}
+    We set the idpattern to none so that $abc doesn't get replaced but ${abc} does.
+
+    Case insensitive.
+    Variables that are found in the buffer, but are not defined as environment variables are ignored.
+    """
+    return Template(buffer).safe_substitute(os.environ, idpattern=None, bracedidpattern="(?a:[_a-z][_a-z0-9]*)")
+
+
+def get_metrics_sink(redis=None):
+    """Creates a clue_metrics CommsQueue on redis for metrics."""
+    from clue.remote.datatypes.queues.comms import CommsQueue
+
+    return CommsQueue("clue_metrics", host=redis)

clue/common/json_utils.py

@@ -0,0 +1,10 @@
+import json
+from typing import Any
+
+
+def try_parse_json(json_str: str, return_raw: bool = False) -> dict[str, Any] | str | None:
+    "Try and parse JSON, optionally returning the raw string if json loading fails."
+    try:
+        return json.loads(json_str)
+    except json.JSONDecodeError:
+        return json_str if return_raw else None

clue/common/list_utils.py

@@ -0,0 +1,11 @@
+from typing import Any
+
+
+def flatten_list(_list: list[list[Any]]):
+    "Flatten a nested list"
+    flat_list = []
+    for sublist in _list:
+        for item in sublist:
+            flat_list.append(item)
+
+    return flat_list