clue-api 1.0.0.dev7__py3-none-any.whl

clue/models/graph.py

@@ -0,0 +1,162 @@
+# ruff: noqa: D101
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from pydantic.alias_generators import to_camel
+
+
+class BaseGraphModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        extra="forbid",
+    )
+
+
+class Operator(StrEnum):
+    equals = "="
+    less_than = "<"
+    greater_than = ">"
+    not_equals = "!="
+    less_than_equal_to = "<="
+    greater_than_equal_to = ">="
+    includes = "~"
+
+
+class Comparator(BaseGraphModel):
+    field: str | None = Field(description="What field should the comparator check against?", default=None)
+    operator: Operator | None = Field(
+        description="What operation should the comparator use for this check?", default=None
+    )
+    value: str | int | bool | float | None = Field(
+        description="What value should the comparator check the field against?", default=None
+    )
+
+
+class DisplayField(Comparator):
+    zoom: float | None = Field(description="What zoom level should the field be shown at?", default=None)
+    label: str = Field(description="What value should be shown from the node in this display field?")
+
+
+class Type(StrEnum):
+    node = "node"
+    edge = "edge"
+
+
+class Style(Comparator):
+    color: str | None = Field(description="What colour should be applied to the matching elements?", default=None)
+    size: float | None = Field(description="What size should the matching elements be?", default=None)
+    type: Type = Field(description="What type of element do you want to select?")
+
+
+class NodeColor(BaseGraphModel):
+    border: str = Field(description="What default colour should the border of the nodes have?", default="grey")
+    center: str = Field(description="What default colour should the center of the nodes have?", default="white")
+
+
+class VisualConfig(BaseGraphModel):
+    text_color: str = Field(description="What colour should the text on the graph be?", default="white")
+    node_color: NodeColor = NodeColor()
+    letter_size: float = Field(description="What size should the text on the graph be?", default=10)
+    letter_width: float = Field(description="What size should the text on the graph be?", default=6.5)
+    x_spacing: float = Field(description="How much spacing should there be between columns in the graph?", default=8)
+    y_spacing: float = Field(description="How much spacing should there be between rows in the graph?", default=20)
+    node_radius: float = Field(description="How large should nodes on the graph be?", default=6)
+    max_arc_radius: float = Field(description="How gradual should curves of the edges of the graph be?", default=8)
+    line_padding_x: float = Field(
+        description="How much horizontal padding should there be between edges on the graph?", default=10
+    )
+    line_padding_y: float = Field(
+        description="How much vertical padding should there be between edges on the graph?", default=10
+    )
+    line_width: float = Field(description="How thick should edges on the graph be?", default=3)
+    enable_entry_padding: bool = Field(
+        description="Should entries be separated by the length of the longest label?", default=True
+    )
+    truncate_labels: bool = Field(description="Should labels be truncated?", default=True)
+    enable_timestamps: bool = Field(
+        description="Should timestamps be enabled? This will also show annotations, if provided", default=True
+    )
+    icon_orientation: Optional[Literal["vertical"] | Literal["horizontal"]] = Field(
+        description="Where should icons be rendered when provided - above or to the side of the node?",
+        default="vertical",
+    )
+    line_direction: Optional[Literal["vertical"] | Literal["horizontal"]] = Field(
+        description="What pathing behaviour should the edges of the graph use by default?",
+        default="horizontal",
+    )
+    row_step: float = Field(
+        description=(
+            "How much spacing should there be between rows? This is calculated dynamically if not "
+            "provided, based on y_spacing"
+        ),
+        default=0,
+    )
+    below_previous: bool = Field(
+        description="Should each column be rendered below the previous?",
+        default=False,
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def prepare_model(cls, data: dict[str, Any]) -> dict[str, str]:  # noqa: ANN102
+        """Calculates the row step if not provided.
+
+        Args:
+            data (dict[str, str]): The data to validate.
+
+        Returns:
+            dict[str, str]: The data including the password.
+        """
+        if "rowStep" not in data:
+            if "ySpacing" in data:
+                data["rowStep"] = 2 * data["ySpacing"]
+            elif "y_spacing" in data:
+                data["rowStep"] = 2 * data["y_spacing"]
+            else:
+                data["rowStep"] = 2 * VisualConfig.model_fields["y_spacing"].default
+
+        return data
+
+
+class Visualization(BaseGraphModel):
+    config: VisualConfig = VisualConfig()
+    type: Literal["tree"] | Literal["cloud"] = Field(
+        description="The type of visualization to use. Defaults to tree", default="tree"
+    )
+
+
+class DisplayConfig(BaseGraphModel):
+    display_field: list[DisplayField] = Field(description="A list of fields to present in the graph.", default=[])
+    styles: list[Style] = Field(description="A list of styles to apply to the graph.", default=[])
+    visualization: Visualization = Visualization()
+
+
+class Node(BaseGraphModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(description="The ID of this node.")
+    edges: list[str] = Field(description="A list of IDs this node connects to.", default=[])
+    markdown: str | None = Field(
+        description="A markdown string providing additional details on this node.", default=None
+    )
+    timestamp: datetime | None = Field(
+        description="The timestamp associated with this node. Only rendered if enable_timestamps is set to true.",
+        default=None,
+    )
+    annotation: str | None = Field(
+        description="The annotation associated with this node. Only rendered if enable_timestamps is set to true.",
+        default=None,
+    )
+    icons: list[str] = Field(description="A list of icons to show next to this node.", default=[])
+
+
+class Metadata(BaseGraphModel):
+    # TODO: Eventually support non-nested data
+    type: Literal["nested"] = "nested"
+    display: DisplayConfig = DisplayConfig()
+    subgraphs: list[Comparator] = Field(
+        description="A list of comparators used to automatically generate subgraphs in the fetcher.", default=[]
+    )

clue/models/model_list.py

@@ -0,0 +1,52 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from clue.models.actions import Action, ActionBase, ActionResult, ActionSpec, ExecuteRequest
+from clue.models.fetchers import FetcherDefinition, FetcherResult
+from clue.models.graph import (
+    BaseGraphModel,
+    Comparator,
+    DisplayConfig,
+    DisplayField,
+    Metadata,
+    Node,
+    NodeColor,
+    Style,
+    VisualConfig,
+)
+from clue.models.network import Annotation, ClueResponse, QueryEntry, QueryResult
+from clue.models.results.graph import GraphResult
+from clue.models.results.image import ImageResult
+from clue.models.selector import Selector
+
+__MODEL_LIST: list[type[BaseModel]] = [
+    Action,
+    ActionBase,
+    ActionResult,
+    ActionSpec,
+    ExecuteRequest,
+    FetcherDefinition,
+    ImageResult,
+    FetcherResult,
+    BaseGraphModel,
+    Comparator,
+    DisplayConfig,
+    DisplayField,
+    GraphResult,
+    Metadata,
+    Node,
+    NodeColor,
+    Style,
+    VisualConfig,
+    ClueResponse,
+    Annotation,
+    QueryEntry,
+    QueryResult,
+    Selector,
+]
+
+MODEL_SCHEMAS: dict[str, dict[str, Any]] = {}
+
+for __model in __MODEL_LIST:
+    MODEL_SCHEMAS[__model.__name__] = __model.model_json_schema()

clue/models/network.py

@@ -0,0 +1,430 @@
+# ruff: noqa: D101
+import hashlib
+import textwrap
+from datetime import datetime, timedelta, timezone
+from email.utils import parseaddr
+from math import floor
+from random import randbytes, sample
+from typing import Any, Literal, Optional, Union
+
+from pydantic import (
+    AliasGenerator,
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationInfo,
+    computed_field,
+    field_validator,
+    model_validator,
+)
+from pydantic_core import Url
+from typing_extensions import Self
+
+from clue.common.logging import get_logger
+from clue.config import CLASSIFICATION, DEBUG, get_version
+from clue.constants.supported_types import SUPPORTED_TYPES
+from clue.models.validators import validate_classification
+
+logger = get_logger(__file__)
+
+
+class ClueResponse(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=AliasGenerator(serialization_alias=lambda field_name: f"api_{field_name}")
+    )
+
+    response: Any = None
+    error_message: Optional[str] = None
+    warning: list[str] = []
+    server_version: str = get_version()
+    status_code: int
+
+
+class Annotation(BaseModel):
+    analytic: Optional[str] = Field(
+        description="Identifier for the analytic producing the knowledge. Mutually exclusive with author.",
+        default=None,
+        examples=["Howler", "Assemblyline", None],
+    )
+    analytic_icon: Optional[str] = Field(
+        description="Formatted string to present an icon for this analytic on the UI using iconify/react format: "
+        "https://iconify.design/docs/icon-components/react/. External icons not yet supported",
+        default=None,
+        examples=["material-symbols:sound-detection-dog-barking", None],
+    )
+    author: Optional[str] = Field(
+        description="The author providing the annotation. Mutually exclusive with analytic.",
+        default=None,
+        examples=["John Smith", None],
+    )
+    quantity: int = Field(
+        description="Number of times this annotation was generated for the given indicator",
+        default=1,
+        examples=[1, 10, 25],
+    )
+    version: Optional[str] = Field(
+        description="The version of the API for the analytic that produced the knowledge",
+        default=None,
+        examples=["v0.0.1", "1.0.0", None],
+    )
+    timestamp: Optional[datetime] = Field(
+        description="A timestamp describing when the knowledge was generated.",
+        default_factory=lambda: datetime.now(timezone.utc),
+        examples=[datetime.now(timezone.utc), datetime.now(timezone.utc) - timedelta(weeks=2)],
+    )
+    type: Literal["opinion", "frequency", "assessment", "mitigation", "context"] = Field(
+        description=textwrap.dedent("""
+                                    What type of annotation is this?
+                                    Opinion (What type of activity is the selector associated with?):
+                                        benign - authorized or harmless activity
+                                        suspect - outlier activity without clear intent
+                                        malicious - intended to cause harm
+                                        obscure - why a reliable opinion might not be possible
+
+                                    Context (Unopinionated facts)
+                                        Examples:
+                                        - Frequency of observation based on summaries
+                                        - Account privileges
+                                        - Sign-in characteristics
+                                        - Geo-location
+                                        - Domain ownership
+                                        - Operation Label
+
+                                    Assessment (An official position)
+                                        Benign Alert Assessments:
+                                        - Ambiguous
+                                        - Security
+                                        - Development
+                                        - False-positive
+                                        - Legitimate
+
+                                        Malicious Alert Assessments:
+                                        - Trivial
+                                        - Recon
+                                        - Attempt
+                                        - Compromise
+                                        - Mitigated
+
+                                    Frequency (A numeric value for the frequency a selector has been seen)
+                                        Higher number - more common selector
+                                        Lower number - less common selector
+
+                                    Mitigation (Suggested actions available to mitigate harm done by the selector)
+                                        exempt - Selector cannot be mitigated, used for known safe selectors
+                                        alertable - Selector can be alerted on
+                                        blockable - Selector can be blocked
+                                        shareable - Selector can be shared with partners/collaborators
+                                        not-alertable - Selector cannot be alerted on
+                                        not-blockable - Selector cannot be blocked
+                                        not-shareable - Selector cannot be shared with partners/collaborators
+                                    """),
+        examples=["opinion", "frequency", "assessment", "mitigation", "context"],
+    )
+    value: Union[str, float, int] = Field(
+        description="The value associated with the type.",
+        examples=[
+            "benign",
+            "suspect",
+            "malicious",
+            "obscure",
+            "IP Located in Canada",
+            "Involved in Operation Cat",
+            11,
+            42.0,
+        ],
+    )
+    confidence: float = Field(
+        description="Self-reported confidence level of the annotation. 0.0 = not confident at all, 1.0 = absolute fact",
+        ge=0.0,
+        le=1.0,
+        examples=[0.0, 0.5, 1.0],
+    )
+    severity: Optional[float] = Field(
+        description="Severity of the annotation, if accurate. 0.0 = not severe at all, 1.0 = extremely important",
+        ge=0.0,
+        le=1.0,
+        default=None,
+        examples=[0.0, 0.5, 1.0, None],
+    )
+    priority: Optional[float] = Field(
+        description=(
+            "What priority to assign to this annotation. Higher priority = more likely to be shown to analysts. "
+            "Optional. If not provided, calculated based on confidence, severity and reliability."
+        ),
+        default=None,
+        examples=[1.0, 50.0, 1000.0, None],
+    )
+    summary: str = Field(
+        description="A plaintext summary of the annotation.",
+        examples=["Example summary of the information in this Annotation"],
+    )
+    details: Optional[str] = Field(
+        description="detailed description of the annotation. Supports markdown formatting.",
+        default=None,
+        examples=["# Here's some annotation details\n\nIt's very interesting", None],
+    )
+    link: Optional[Url] = Field(
+        description="Link for more information about this specific annotation",
+        default=None,
+        examples=[Url("https://example.com/annotation"), None],
+    )
+    icon: Optional[str] = Field(
+        description="Formatted string to present an icon for this annotation on the UI using iconify/react format: "
+        "https://iconify.design/docs/icon-components/react/. External icons not yet supported",
+        default=None,
+        examples=["material-symbols:sound-detection-dog-barking", None],
+    )
+    ubiquitous: bool = Field(
+        description="Does this annotation show up on the vast majority of selectors (i.e. asset provenance, "
+        "organization ownership/non-ownership of IP address)",
+        default=False,
+        examples=[True, False],
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def reliability(self: Self) -> Optional[float]:
+        """Accurately calculated reliability of the annotation. 0.0 = not reliable, 1.0 = extremely reliable"""
+        if self.author:
+            return 1.0
+
+        # TODO: Implement a way to set reliability of annotations from analytics
+        return None
+
+    @model_validator(mode="after")
+    def validate_model(self: Self) -> Self:  # noqa: C901
+        """Validates the entire model.
+
+        Raises:
+            AssertionError: Raised whenever a field is invalid on the model.
+
+        Returns:
+            Self: The validated model.
+        """
+        if not (self.author or self.analytic):
+            raise AssertionError("Author or analytic must be set.")
+        if self.author and self.analytic:
+            raise AssertionError("Author and analytic are mutually exclusive.")
+
+        if self.type in ["opinion", "assessment", "mitigation", "context"]:
+            if not isinstance(self.value, str):
+                raise AssertionError(
+                    f"Value must be a string if type is not frequency. Type is ({type(self.value).__name__})"
+                )
+        else:
+            try:
+                self.value = int(self.value)
+            except Exception as e:
+                raise AssertionError("Value must be an int if type is frequency.") from e
+
+        if self.type == "opinion":
+            valid_options = ["benign", "suspicious", "malicious", "obscure"]
+            if self.value not in valid_options:
+                raise AssertionError(
+                    f"If type is opinion, value must be one of ({', '.join(valid_options)}). Value is {self.value}"
+                )
+
+        elif self.type == "mitigation":
+            valid_options = [
+                "exempt",
+                "alertable",
+                "blockable",
+                "shareable",
+                "not-alertable",
+                "not-blockable",
+                "not-shareable",
+            ]
+            if self.value not in valid_options:
+                raise AssertionError(
+                    f"If type is mitigation, value must be one of ({', '.join(valid_options)}). Value is {self.value}"
+                )
+
+        elif self.type == "assessment":
+            valid_options = [
+                "ambiguous",
+                "security",
+                "development",
+                "false-positive",
+                "legitimate",
+                "trivial",
+                "recon",
+                "attempt",
+                "compromise",
+                "mitigated",
+            ]
+            if self.value not in valid_options:
+                raise AssertionError(
+                    f"If type is assessment, value must be one of ({', '.join(valid_options)}). Value is {self.value}"
+                )
+
+        if self.icon and self.type != "context":
+            raise AssertionError("Icons are currently only supported for 'context' annotations.")
+
+        if self.icon and len(self.icon.split(":")) != 2:
+            raise AssertionError("Icon field not formatted correctly. Must be in the format <icon_type>:<icon_id>.")
+
+        if self.analytic_icon and len(self.analytic_icon.split(":")) != 2:
+            raise AssertionError(
+                "Analytic Icon field not formatted correctly. Must be in the format <icon_type>:<icon_id>."
+            )
+
+        if not self.priority and self.severity:
+            modifier = self.reliability if self.reliability is not None else self.confidence
+
+            # TODO: Tweak this behaviour as we have a better idea how it should behave.
+            # Always outputs a priority in the range [0, 1]
+            self.priority = modifier * (self.severity ** (2 - modifier))
+
+        return self
+
+
+class QueryEntry(BaseModel):
+    classification: str = Field(
+        description="Classification of results by the enrichment",
+        default="TLP:CLEAR",
+        examples=sample(sorted(CLASSIFICATION.list_all_classification_combinations()), k=5),
+    )
+    count: int = Field(
+        description="Number of matches from the search",
+        default=1,
+        examples=sorted([floor(i / 10) for i in randbytes(5)]),  # noqa: S311
+    )
+    link: Optional[Url] = Field(
+        description="Link to more information", default=None, examples=[Url("https://example.com/moreinfo"), None]
+    )
+    annotations: list[Annotation] = Field(
+        description="A list of annotations returned from the service for this entry", default=[]
+    )
+    raw_data: Any = Field(
+        description="The raw records associated with the generated annotations.",
+        default=None,
+        examples=[{"id": 1, "raw_field": "some_data"}, [{"id": 1, "other_data": "example", "other_row": 45}]],
+    )
+
+    model_config = ConfigDict(validate_assignment=True)
+
+    @field_validator("classification")
+    @classmethod
+    def check_classification(cls, classification: str) -> str:  # noqa: ANN102
+        """Validates the provided classification.
+
+        Args:
+            classification (str): The classification to validate.
+
+        Raises:
+            AssertionError: Raised whenever the provided classification is not valid.
+
+        Returns:
+            str: The validated classification.
+        """
+        return validate_classification(classification)
+
+
+class QueryResult(BaseModel):
+    type: str = Field(
+        description="The type of the value represented by this result", examples=list(SUPPORTED_TYPES.keys())
+    )
+    value: str = Field(
+        description="The value represented by this result",
+        examples=["127.0.0.1", "email@example.com", hashlib.sha256("example".encode()).hexdigest()],
+    )
+    source: str = Field(description="The name of the plugin providing this result", examples=["example_plugin"])
+    error: Optional[str] = Field(
+        description="Error message returned by data source",
+        default=None,
+        examples=["An error occurred when enriching the data.", None],
+    )
+    items: list[QueryEntry] = Field(description="List of results from the source", default=[])
+    maintainer: Optional[str] = Field(
+        description="Email contact in the RFC-5322 format 'Full Name <email_address>'.",
+        default=None,
+        examples=["maintainer@example.com", None],
+    )
+    datahub_link: Optional[Url] = Field(
+        description="Link to datahub entry on this enrichment",
+        default=None,
+        examples=[Url("https://example.com/datahub"), None],
+    )
+    documentation_link: Optional[Url] = Field(
+        description="Link to documentation on this enrichment",
+        default=None,
+        examples=[Url("https://example.com/documentation"), None],
+    )
+    latency: float = Field(
+        description="Total duration (in milliseconds) taken to resolve this result",
+        default=0,
+        examples=sorted([i * 10 for i in randbytes(5)]),  # noqa: S311
+    )
+
+    model_config = ConfigDict(validate_assignment=True)
+
+    @field_validator("maintainer")
+    @classmethod
+    def validate_maintainer(cls, maintainer: Optional[str]) -> Optional[str]:  # noqa: ANN102
+        """Validates the maintainer field.
+
+        Args:
+            maintainer (Optional[str]): The maintainer field to validate. If None, it will be passed through.
+
+        Raises:
+            AssertionError: Raised whenever the field is in an invalid format.
+
+        Returns:
+            Optional[str]: The validated maintainer field.
+        """
+        if maintainer:
+            parsed_addr = parseaddr(maintainer)
+            if not (all(parsed_addr) and "@" in parsed_addr[1]):
+                raise AssertionError("Maintainer string must be in RFC-5322 format.")
+
+        return maintainer
+
+    @field_validator("items")
+    @classmethod
+    def validate_items(cls, items: list[QueryEntry], info: ValidationInfo) -> list[QueryEntry]:  # noqa: ANN102
+        """Validate that if classification data was provided, all annotations match the user's classification.
+
+        Args:
+            items (list[QueryEntry]): The items to validate.
+            info (ValidationInfo): Additional validation info.
+
+        Returns:
+            list[QueryEntry]: The validated items field.
+        """
+        if info.context:
+            user_classification = info.context.get("user", {}).get("classification", None)
+            if user_classification:
+                filtered_results: list[QueryEntry] = []
+
+                for item in items:
+                    if CLASSIFICATION.is_accessible(user_classification, item.classification):
+                        filtered_results.append(item)
+                    else:
+                        logger.debug(
+                            "Removing item at classification %s, inaccessible to user classification %s",
+                            item.classification,
+                            user_classification,
+                        )
+
+                if len(items) > len(filtered_results):
+                    logger.info(
+                        "Dropped %s items due to inaccessible classification (user classification: %s)",
+                        len(items) - len(filtered_results),
+                        user_classification,
+                    )
+                elif DEBUG:
+                    logger.debug(
+                        "All %s values are accessible by user classification %s", len(items), user_classification
+                    )
+
+                return filtered_results
+            else:
+                logger.warning("No user classification given, classification parsing will not occur")
+        else:
+            logger.warning("No user context given, classification parsing will not occur")
+
+        return items
+
+
+class PluginResponse(BaseModel):
+    pass

clue/models/results/__init__.py

@@ -0,0 +1,34 @@
+# ruff: noqa: D101
+from typing import Any, TypeVar
+
+from pydantic import BaseModel
+
+from clue.models.results.base import Result
+from clue.models.results.graph import GraphResult
+from clue.models.results.image import ImageResult
+from clue.models.results.status import StatusResult
+
+DATA = TypeVar("DATA", bound=(dict[str, Any] | list[dict[str, Any]] | str | Result))
+
+FORMAT_MAPPINGS: dict[type[BaseModel] | type[dict] | type[list] | type[str], str] = {
+    dict: "json",
+    list: "json",
+    str: "markdown",
+    ImageResult: ImageResult.format(),
+    StatusResult: StatusResult.format(),
+    GraphResult: GraphResult.format(),
+}
+FORMAT_MAPPINGS_REVERSE: dict[str, list[type[BaseModel] | type[dict] | type[list] | type[str]]] = {}
+
+for k, v in FORMAT_MAPPINGS.items():
+    if v not in FORMAT_MAPPINGS_REVERSE:
+        FORMAT_MAPPINGS_REVERSE[v] = [k]
+    else:
+        FORMAT_MAPPINGS_REVERSE[v].append(k)
+
+
+def register_result(model: type[Result]):
+    "Add a new result type to the mappings"
+    if model.format() not in FORMAT_MAPPINGS_REVERSE:
+        FORMAT_MAPPINGS[model] = model.format()
+        FORMAT_MAPPINGS_REVERSE[model.format()] = [model]

clue/models/results/base.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+
+
+class Result(BaseModel):
+    "Base result object"
+
+    @staticmethod
+    def format() -> str:
+        "Return the format of the result"
+        raise NotImplementedError()