psengine 2.0.4__py3-none-any.whl

psengine/stix2/enriched_indicator.py

@@ -0,0 +1,261 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+
+import stix2
+
+from ..helpers import FormattingHelpers
+from .base_stix_entity import BaseStixEntity
+from .complex_entity import IndicatorEntity, NoteEntity, Relationship
+from .errors import STIX2TransformError, UnsupportedConversionTypeError
+from .helpers import convert_entity
+from .simple_entity import TTP, ThreatActor
+
+LOG = logging.getLogger(__name__)
+
+
+class EnrichedIndicator(IndicatorEntity):
+    """Class for converting Indicator + risk score + links to OpenCTI bundle."""
+
+    def __init__(
+        self,
+        name: str,
+        type_: str,
+        evidence_details: list,
+        link_hits: list = None,
+        risk_mapping: list = None,
+        ai_insights: dict = None,
+        author: stix2.Identity = None,
+        confidence: int = None,
+        create_indicator: bool = True,
+        create_obs: bool = True,
+        tlp_marking='amber',
+    ):
+        """Indicator container. Containers Indicator, observable, and relationship between them.
+
+        Args:
+            name (str): Indicator value
+            type_ (str): Recorded Future type of indicator
+            evidence_details (list): risk rules + evidence details
+            link_hits (list, optional): list of lists
+            risk_mapping (list, optional): Risk rule to TTP mapping
+            ai_insights (dict, optional): AI insights for IOC in Recorded Future
+            author (stix2.Identity, optional): Recorded Future Identity
+            confidence (int, optional): Confidence score of indicator
+            create_indicator (bool, optional): flag that governs if indicator should be created
+            create_obs (bool, optional): flag that governs if observable should be created
+            tlp_marking (str, optional): the TLP level. Default to amber
+
+        Raises:
+            STIX2TransformError: if transformation fails
+        """
+        link_hits = link_hits or []
+        risk_mapping = risk_mapping or []
+        ai_insights = ai_insights or {}
+        evidence_details = [e if isinstance(e, dict) else e.json() for e in evidence_details]
+        labels = ['rf:' + (rule.get('Rule') or rule.get('rule')) for rule in evidence_details]
+
+        description = self._format_description(ai_insights)
+        super().__init__(
+            name=name,
+            type_=type_,
+            confidence=confidence,
+            create_indicator=create_indicator,
+            create_obs=create_obs,
+            labels=labels,
+            description=description,
+            tlp_marking=tlp_marking,
+            author=author,
+        )
+
+        self._add_risk_score_as_note(confidence)
+
+        for ttp in risk_mapping:
+            self._add_ttp(ttp)
+        for rule in evidence_details:
+            self._add_rule(rule)
+
+        for hit in link_hits:
+            self._add_link(hit)
+
+    def _format_description(self, ai_insights: dict):
+        """If AI insights are available then:
+            - use it as a description
+            - otherwise use a default string value.
+
+        Args:
+            ai_insights (dict): string from RF API
+
+        Returns:
+            str: Indicator description
+        """
+        ai_insights = ai_insights if isinstance(ai_insights, dict) else ai_insights.json()
+        description = 'Indicator from Recorded Future'
+        if ai_insights.get('text') is not None:
+            description = '### Recorded Future AI Insights\n\n{}'.format(
+                FormattingHelpers.cleanup_ai_insights(ai_insights.get('text')),
+            )
+
+        return description
+
+    def _add_link(self, hit) -> None:
+        hit = hit if isinstance(hit, dict) else hit.json()
+        for section in hit['sections']:
+            for list_ in section['lists']:
+                if list_['type']['name'] == 'Threat Actor':
+                    for entity in list_['entities']:
+                        stix_entity = ThreatActor(entity['name'])
+                        self.stix_objects.append(stix_entity.stix_obj)
+                        self._relate(stix_entity)
+                else:
+                    for entity in list_['entities']:
+                        try:
+                            stix_entity = convert_entity(entity['name'], entity['type'])
+                        except UnsupportedConversionTypeError as err:
+                            LOG.warning(str(err) + '. Skipping...')
+                            continue
+                        if isinstance(stix_entity, IndicatorEntity):
+                            self.stix_objects.extend(stix_entity.stix_objects)
+                            self._relate(stix_entity)
+                        else:
+                            self.stix_objects.append(stix_entity.stix_obj)
+                            self._relate(stix_entity)
+
+    def _relate(self, obj: BaseStixEntity) -> None:
+        """Creates relationship between object and indicator/observabe.
+
+        Args:
+            obj (RFBaseStixEntity): Stix object we're linking to
+
+        Raises:
+            STIX2TransformError: Generic transofmr error
+        """
+        if isinstance(obj, IndicatorEntity):
+            sources = []
+            targets = []
+            if self.indicator:
+                sources.append(self.indicator)
+            if self.observable:
+                sources.append(self.observable)
+            if obj.indicator:
+                targets.append(obj.indicator)
+            if obj.observable:
+                targets.append(obj.observable)
+            self._append_indicator_relationships(sources, targets)
+        elif isinstance(obj, BaseStixEntity):
+            if self.indicator:
+                self.stix_objects.append(
+                    Relationship(
+                        source=self.indicator.id,
+                        target=obj.stix_obj.id,
+                        type_='indicates',
+                        author=self.author,
+                    ).stix_obj,
+                )
+            if self.observable:
+                self.stix_objects.append(
+                    Relationship(
+                        source=self.observable.id,
+                        target=obj.stix_obj.id,
+                        type_='related-to',
+                        author=self.author,
+                    ).stix_obj,
+                )
+        else:
+            raise STIX2TransformError('Cannot transform, entity is not of correct type')
+
+    def _append_indicator_relationships(self, sources, targets) -> None:
+        for source in sources:
+            for target in targets:
+                self.stix_objects.append(
+                    Relationship(
+                        source=source.id,
+                        target=target.id,
+                        type_='related-to',
+                        author=self.author,
+                    ).stix_obj,
+                )
+
+    def _add_ttp(self, ttp: dict) -> None:
+        """Adds a TTP from risk mapping.
+
+        Args:
+            ttp (dict): maps a risk rule to one or more T codes
+        """
+        ttp = ttp if isinstance(ttp, dict) else ttp.json()
+        for cat in ttp.get('categories', []):
+            if cat['framework'] == 'MITRE':
+                obj = TTP(cat['name'])
+                self.stix_objects.append(obj.stix_obj)
+                self._relate(obj)
+
+    def _add_rule(self, rule: dict) -> None:
+        """Convert risk rule + evidence detail to notes.
+
+        Args:
+            rule (dict): Risk rule + evidence details json blobs
+        """
+        rule = rule if isinstance(rule, dict) else rule.json()
+        refs = []
+        if self.indicator:
+            refs.append(self.indicator.id)
+        if self.observable:
+            refs.append(self.observable.id)
+
+        # in risklists and enrichment 'rule' has different capitalization
+        rule_name = rule.get('Rule') or rule.get('rule')
+
+        if not rule_name:
+            raise STIX2TransformError('Cannot transform, rule name is missing')
+
+        self.stix_objects.append(
+            NoteEntity(
+                name=rule_name,
+                content=(rule.get('evidenceString') or rule.get('EvidenceString')),
+                object_refs=refs,
+                author=self.author,
+            ).stix_obj,
+        )
+
+    def _add_risk_score_as_note(self, risk_score: int) -> None:
+        """Add Confidende/Risk Score as a note.
+
+        Args:
+            risk_score (int): Confidence score
+        """
+        if not risk_score:
+            raise STIX2TransformError('Cannot transform, confidence is missing')
+        object_refs = []
+        if self.indicator:
+            object_refs.append(self.indicator.id)
+        if self.observable:
+            object_refs.append(self.observable.id)
+        self.stix_objects.append(
+            NoteEntity(
+                name='Recorded Future Risk Score',
+                content=f'{risk_score}/99',
+                object_refs=object_refs,
+                author=self.author,
+            ).stix_obj,
+        )
+
+    @property
+    def bundle(self) -> stix2.v21.Bundle:
+        """Creates STIX2 bundle.
+
+        Returns:
+            stix2.v21.Bundle: Bundle
+        """
+        self.stix_objects.append(self.author)
+        return stix2.v21.Bundle(objects=self.stix_objects, allow_custom=True)

psengine/stix2/errors.py

@@ -0,0 +1,22 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from ..errors import RecordedFutureError
+
+
+class STIX2TransformError(RecordedFutureError):
+    """Error raised when invalid parameters are passed to the STIX2 module."""
+
+
+class UnsupportedConversionTypeError(STIX2TransformError):
+    """Error raised when client tries to convert an Recorded FUture type that is not supported."""

psengine/stix2/helpers.py

@@ -0,0 +1,68 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from .complex_entity import IndicatorEntity
+from .constants import CONVERT_ENTITY_KWARGS, INDICATOR_TYPES
+from .errors import STIX2TransformError, UnsupportedConversionTypeError
+from .simple_entity import TTP, Identity, Malware, Vulnerability
+
+SIMPLE_ENTITY_MAP = {
+    'MitreAttackIdentifier': TTP,
+    'Company': Identity,
+    'Person': Identity,
+    'Organization': Identity,
+    'Malware': Malware,
+    'CyberVulnerability': Vulnerability,
+}
+
+
+def convert_entity(
+    name: str,
+    entity_type: str,
+    create_indicator: bool = True,
+    create_obs: bool = False,
+    **kwargs,
+):
+    """Converts RF entity to STIX2.
+
+    Args:
+        name (str): Name of entity
+        entity_type (str): Recorded Future type of entity
+        create_indicator (bool, optional): flag to determine if create indicator
+        create_obs (bool, optional): flag to determine if create observable
+        **kwargs: Any other fields that can be used in an entity.
+                  Must be one of CONVERT_ENTITY_KWARGS
+
+    No Longer Returned:
+        A subclass of RFBaseEntity.
+    """
+    for key in kwargs:
+        if key not in CONVERT_ENTITY_KWARGS:
+            msg = f'{key} is invalid keyword argument for convert_entity. Only {CONVERT_ENTITY_KWARGS} are accepted'  # noqa: E501
+            raise STIX2TransformError(msg)
+    if entity_type in INDICATOR_TYPES:
+        return IndicatorEntity(
+            name=name,
+            type_=entity_type,
+            create_indicator=create_indicator,
+            create_obs=create_obs,
+        )
+    elif entity_type in SIMPLE_ENTITY_MAP:
+        ent = SIMPLE_ENTITY_MAP[entity_type]
+        if ent == Identity:
+            return ent(name, rf_type=entity_type)
+        return SIMPLE_ENTITY_MAP[entity_type](name, **kwargs)
+    else:
+        raise UnsupportedConversionTypeError(
+            f'Could not convert entity {name} because type {entity_type} is not supported',
+        )

psengine/stix2/rf_bundle.py

@@ -0,0 +1,240 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import json
+import logging
+
+from stix2 import Bundle, Identity, Report
+from stix2.exceptions import (
+    ExtraPropertiesError,
+    InvalidValueError,
+    MissingPropertiesError,
+    STIXError,
+)
+
+from ..analyst_notes import AnalystNote
+from ..risklists.models import DefaultRiskList
+from .complex_entity import DetectionRuleEntity, IndicatorEntity
+from .constants import ENTITY_TYPE_MAP, REPORT_TYPE_MAPPER, SUPPORTED_HUNTING_RULES, TLP_MAP
+from .enriched_indicator import EnrichedIndicator
+from .errors import STIX2TransformError, UnsupportedConversionTypeError
+from .helpers import convert_entity
+from .util import create_rf_author
+
+LOG = logging.getLogger(__name__)
+
+
+class RFBundle:
+    """Class for creating STIX2 bundles from Recorded Future objects."""
+
+    @classmethod
+    def from_default_risklist(
+        cls,
+        risklist: list[DefaultRiskList],
+        entity_type: str,
+        identity: Identity = None,
+    ) -> Bundle:
+        """Creates STIX2 bundle from a Recorded Future default risklist.
+
+        Args:
+            risklist (str): Recorded Future default risklist (contains the standard 5 columns)
+            entity_type (str): entity type
+            identity (_type_, optional): Author identity. Defaults to Recorded Future
+
+        Raises:
+            STIX2TransformError: if the risklist is not valid
+            STIX2TransformError: if EvidenceDetails is not valid JSON
+            STIX2TransformError: if the bundle cannot be created
+
+        Returns:
+            Bundle: STIX2 bundle
+        """
+        if not identity:
+            identity = create_rf_author()
+        objects = [identity]
+        LOG.info(f'Creating STIX2 bundle from {entity_type} risklist')
+
+        try:
+            enriched_indicators = []
+            for ioc in risklist:
+                indicator = EnrichedIndicator(
+                    name=ioc.ioc,
+                    type_=ENTITY_TYPE_MAP[entity_type],
+                    confidence=ioc.risk_score,
+                    evidence_details=ioc.evidence_details,
+                )
+                enriched_indicators.append(indicator)
+
+        except KeyError as ke:
+            raise STIX2TransformError(f'Risklist missing header: {ke}') from ke
+
+        except json.JSONDecodeError as jse:
+            raise STIX2TransformError(f'EvidenceDetails is not valid JSON: {jse}') from jse
+
+        for i in enriched_indicators:
+            objects.extend(i.stix_objects)
+
+        try:
+            bundle = Bundle(objects=objects, allow_custom=True)
+        except (
+            ValueError,
+            ExtraPropertiesError,
+            InvalidValueError,
+            MissingPropertiesError,
+            STIXError,
+        ) as e:
+            raise STIX2TransformError(f'Failed to create STIX2 bundle from risklist. {e}') from e
+
+        return bundle
+
+    @classmethod
+    def from_analyst_note(
+        cls,
+        note: AnalystNote,
+        attachment: bytes = None,
+        split_snort: bool = False,
+        identity: Identity = None,
+    ) -> Bundle:
+        """Creates a STIX2 bundle from a Recorded Future analyst note.
+
+        Args:
+            note (AnalystNote): Recorded Future analyst note
+            attachment (bytes, optional): Note attachment. Defaults to None.
+            split_snort (bool, optional): Split snort rules into separate DetectionRule objects.
+            Defaults to False.
+            identity (stix2.Identity, optional): Author. Defaults to Recorded Future.
+
+        Returns:
+            Bundle: STIX2 bundle
+        """
+        LOG.info(f'Creating STIX2 bundle from analyst note {note.id_}')
+
+        if not identity:
+            identity = create_rf_author()
+        objects = [identity]
+        topics = [topic.name for topic in note.attributes.topic]
+        report_types = _create_report_types(topics)
+        for entity in note.attributes.note_entities:
+            try:
+                stix_entity = convert_entity(entity.name, entity.type_)
+                if isinstance(stix_entity, IndicatorEntity):
+                    objects.extend(stix_entity.stix_objects)
+                else:
+                    objects.append(stix_entity.stix_obj)
+
+            except UnsupportedConversionTypeError as err:  # noqa: PERF203
+                LOG.warning(str(err) + '. Skipping...')
+                continue
+
+        if attachment and note.detection_rule_type in SUPPORTED_HUNTING_RULES:
+            # This is a workaround for OpenCTI
+            # OpenCTI does not support multiple Snort rules within a single DetectionRule object
+            # so we split them into separate objects (split_snort = True)
+            if note.detection_rule_type == 'snort' and split_snort is True:
+                objects.extend(_split_snort_rules(note, str(attachment, 'UTF-8')))
+            else:
+                rule = DetectionRuleEntity(
+                    name=note.attributes.title,
+                    type_=note.detection_rule_type,
+                    content=str(attachment, 'UTF-8'),
+                )
+                objects.append(rule.stix_obj)
+
+        external_references = _generate_external_references(note.attributes.validation_urls)
+        external_references.append(
+            {
+                'source_name': 'Recorded Future',
+                'url': note.portal_url,
+            },
+        )
+
+        report = Report(
+            name=note.attributes.title,
+            created_by_ref=identity.id,
+            description=note.attributes.text,
+            published=note.attributes.published,
+            labels=topics,
+            report_types=report_types,
+            object_refs=[obj.id for obj in objects],
+            object_marking_refs=TLP_MAP['amber'],
+            external_references=external_references,
+        )
+        objects.append(report)
+
+        bundle = Bundle(objects=objects, allow_custom=True)
+
+        return bundle
+
+
+# Helpers for bundle creation
+
+
+def _split_snort_rules(note: AnalystNote, attachment: str) -> list:
+    """Splits snort rules into multiple DetectionRule objects.
+
+    Args:
+        note (AnalystNote): AnalystNote object
+        attachment (str): snort rule payload
+    """
+    rules = []
+    ctr = 1
+    temp_description = []
+    for line in attachment.split('\n'):
+        line = line.strip()
+
+        # skip comments and empty lines
+        if line.startswith('#') or not line:
+            temp_description.append(line[1:].strip())
+            continue
+        rules.append(
+            DetectionRuleEntity(
+                name=note.attributes.title + f', Rule {ctr}',
+                type_=note.detection_rule_type,
+                content=line,
+                description='\n'.join(temp_description),
+            ).stix_obj,
+        )
+        # reset description for next rule
+        temp_description = []
+        ctr += 1
+    return rules
+
+
+def _create_report_types(topics: list) -> list:
+    """Map topics to STIX2 report types.
+
+    Args:
+        topics (list): List of topics to map
+
+    Returns:
+        list: List of STIX2 report types
+    """
+    ret = set()
+    for topic in topics:
+        if topic not in REPORT_TYPE_MAPPER:
+            LOG.warning(f'Could not map a report type for type {topic}')
+            continue
+        ret.add(REPORT_TYPE_MAPPER[topic])
+    return list(ret)
+
+
+def _generate_external_references(urls: list) -> list:
+    """Generate External references from validation urls."""
+    refs = []
+    if urls is None:
+        return refs
+
+    for url in urls:
+        source_name = url.name.split('/')[2].split('.')[-2].capitalize()
+        refs.append({'source_name': source_name, 'url': url})
+    return refs