psengine 2.0.4__py3-none-any.whl

psengine/__init__.py

@@ -0,0 +1,22 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+
+from ._version import __version__ as version
+from .base_http_client import BaseHTTPClient
+from .errors import ReadFileError, RecordedFutureError, WriteFileError
+from .rf_client import RFClient
+
+log = logging.getLogger('psengine')
+log.addHandler(logging.NullHandler())

psengine/_sdk_id.py

@@ -0,0 +1,16 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from ._version import __version__
+
+SDK_ID = f'psengine-py/{__version__}'

psengine/_version.py

@@ -0,0 +1,14 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+__version__ = '2.0.4'

psengine/analyst_notes/__init__.py

@@ -0,0 +1,32 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from .errors import (
+    AnalystNoteAttachmentError,
+    AnalystNoteDeleteError,
+    AnalystNoteError,
+    AnalystNoteLookupError,
+    AnalystNotePreviewError,
+    AnalystNotePublishError,
+    AnalystNoteSearchError,
+)
+from .helpers import save_attachment, save_note
+from .note import (
+    AnalystNote,
+    AnalystNotePreviewIn,
+    AnalystNotePreviewOut,
+    AnalystNotePublishIn,
+    AnalystNotePublishOut,
+    AnalystNoteSearchIn,
+)
+from .note_mgr import AnalystNoteMgr

psengine/analyst_notes/constants.py

@@ -0,0 +1,15 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+URL_TO_PORTAL = 'https://app.recordedfuture.com/portal/analyst-note/shared/true/{}'
+NOTES_PER_PAGE = 20

psengine/analyst_notes/errors.py

@@ -0,0 +1,42 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from ..errors import RecordedFutureError
+
+
+class AnalystNoteError(RecordedFutureError):
+    """Error raise when the init of AnalystNote is failing."""
+
+
+class AnalystNoteLookupError(RecordedFutureError):
+    """Error raise when cannot lookup an analyst note."""
+
+
+class AnalystNoteSearchError(RecordedFutureError):
+    """Error raise when cannot search analyst notes."""
+
+
+class AnalystNoteAttachmentError(RecordedFutureError):
+    """Error raise when cannot lookup an analyst note."""
+
+
+class AnalystNoteDeleteError(RecordedFutureError):
+    """Error raise when cannot delete an analyst note."""
+
+
+class AnalystNotePreviewError(RecordedFutureError):
+    """Error raise when cannot post to preview endpoint."""
+
+
+class AnalystNotePublishError(RecordedFutureError):
+    """Error raise when cannot post to publish endpoint."""

psengine/analyst_notes/helpers.py

@@ -0,0 +1,90 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import json
+import logging
+from pathlib import Path
+from typing import Union
+
+from pydantic import validate_call
+
+from ..errors import WriteFileError
+from ..helpers import OSHelpers, debug_call
+from .note import AnalystNote
+
+LOG = logging.getLogger('psengine.analyst_notes.helpers')
+
+
+@debug_call
+@validate_call
+def save_attachment(
+    note_id: str, data: Union[bytes, str], ext: str, output_directory: Union[str, Path]
+) -> None:
+    """Save yara, sigma, snort or pdf to file. The file will use the extension provided and the
+    ``note_id`` to create the filename.
+
+    Args:
+        note_id (str): ``AnalystNote`` id
+        data (Union[bytes, str]): data, returned from ``fetch_attachment``
+        ext (str): extension of the attachment, returned by ``fetch_attachment``
+        output_directory (str, Path): the directory to save the file into
+    """
+    output_directory = (
+        output_directory if isinstance(output_directory, str) else output_directory.as_posix()
+    )
+    _save_attachment(note_id, data, ext, output_directory)
+
+
+@debug_call
+@validate_call
+def save_note(note: AnalystNote, output_directory: Union[str, Path]) -> None:
+    """Save AnalystNote object on file. The file will have the name of the note id.
+
+    Args:
+        note (AnalystNote): note to save.
+        output_directory (str, Path): the directory to save the file into
+    """
+    output_directory = (
+        output_directory if isinstance(output_directory, str) else output_directory.as_posix()
+    )
+    _save_attachment(
+        note_id=note.id_,
+        data=json.dumps(note.json(), indent=4),
+        ext='json',
+        output_directory=output_directory,
+    )
+
+
+def _save_attachment(
+    note_id: str, data: Union[bytes, str], ext: str, output_directory: str
+) -> None:
+    """Save attachment from bytes or note itself from json.
+
+    Args:
+        note_id (str): id of the note, will be the filename
+        data (bytes | str): content to save
+        ext (str): extension of the file.
+        output_directory (str): the directory to save the file into
+
+    Raises:
+        WriteFileError: if saving to file fails
+    """
+    try:
+        note_id = note_id.removeprefix('doc:')
+        LOG.debug(f"Saving file related to '{note_id}' to disk")
+
+        dir_path = OSHelpers.mkdir(output_directory)
+        note_path = Path(dir_path) / f'{note_id}.{ext}'
+        note_path.write_bytes(data) if isinstance(data, bytes) else note_path.write_text(data)
+    except (FileNotFoundError, IsADirectoryError, PermissionError, OSError) as err:
+        raise WriteFileError(f'Failed to save file to disk. Cause: {err.args}') from err

psengine/analyst_notes/models.py

@@ -0,0 +1,219 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+from datetime import datetime
+from typing import Any, Optional, Union
+
+from pydantic import Field, ValidationError, field_validator, model_validator
+
+from ..common_models import IdNameType, IdNameTypeDescription, RFBaseModel
+
+
+class DiamondModel(RFBaseModel):
+    start: Optional[datetime] = None
+    stop: Optional[datetime] = None
+    malicious_infrastructure: Optional[list[IdNameTypeDescription]] = []
+    capabilities: Optional[list[IdNameTypeDescription]] = []
+    adversary: Optional[list[IdNameTypeDescription]] = []
+    target: Optional[list[IdNameTypeDescription]] = []
+
+
+class Query(RFBaseModel):
+    title: str
+    url: Optional[IdNameTypeDescription] = None
+
+
+class Position(RFBaseModel):
+    longitude: float
+    latitude: float
+
+
+class PositionEvent(RFBaseModel):
+    start: datetime
+    stop: datetime
+    location: Optional[list[IdNameTypeDescription]] = []
+    event_positions: Optional[list[Position]] = []
+
+
+class CyberAttackEvent(RFBaseModel):
+    start: datetime
+    stop: datetime
+    adversary: Optional[list[IdNameTypeDescription]] = []
+    target: Optional[list[IdNameTypeDescription]] = []
+    capabilities: list[IdNameTypeDescription] = []
+    malicious_infrastructure: Optional[list[IdNameTypeDescription]] = []
+    operation: Optional[list[IdNameTypeDescription]] = []
+
+
+class ArmedConflictEvent(PositionEvent):
+    attacker: Optional[list[IdNameTypeDescription]] = []
+    target: Optional[list[IdNameTypeDescription]] = []
+
+
+class ArmsPurchaseSaleEvent(RFBaseModel):
+    start: datetime
+    stop: datetime
+    arms_seller: Optional[list[IdNameTypeDescription]] = []
+    arms_purchaser: Optional[list[IdNameTypeDescription]] = []
+
+
+class DiseaseOutbreakEvent(PositionEvent):
+    disease: Optional[list[IdNameTypeDescription]] = []
+    facility: Optional[list[IdNameTypeDescription]] = []
+
+
+class EnvironmentalIssueEvent(PositionEvent):
+    environmental_issue: list[str]
+
+
+class ManMadeDisasterEvent(PositionEvent):
+    facility: list[IdNameTypeDescription]
+    manmade_disaster: Union[list[IdNameTypeDescription], list[str]]
+
+
+class MilitaryManeuverEvent(PositionEvent):
+    actors: Optional[list[IdNameTypeDescription]] = []
+
+
+class NaturalDisasterEvent(PositionEvent):
+    natural_disaster: list[IdNameTypeDescription]
+
+
+class NuclearMaterialTransactionEvent(PositionEvent):
+    material: list[str]
+    location_origin: Optional[list[str]] = []
+    location_destination: Optional[list[str]] = []
+
+
+class PersonThreatEvent(RFBaseModel):
+    start: datetime
+    stop: datetime
+    threatened: list[IdNameTypeDescription]
+    actor: Optional[list[IdNameTypeDescription]] = []
+
+
+class ProtestEvent(RFBaseModel):
+    protest_target: Optional[list[IdNameTypeDescription]] = []
+
+
+class MalwareAnalysisEvent(RFBaseModel):
+    start: datetime
+    stop: datetime
+    malware: list[IdNameTypeDescription]
+    attacker: Optional[list[IdNameTypeDescription]] = []
+    malicious_infrastructure: Optional[list[IdNameTypeDescription]] = []
+    ttp: Optional[list[IdNameTypeDescription]] = []
+    target: Optional[list[IdNameTypeDescription]] = []
+    exploit: Optional[list[IdNameTypeDescription]] = []
+    hash_: Optional[list[IdNameTypeDescription]] = Field(alias='hash', default=[])
+
+
+ATTRIBUTES_MAPPING = {
+    'ArmedConflict': ArmedConflictEvent,
+    'ArmsPurchaseSale': ArmsPurchaseSaleEvent,
+    'Coup': PositionEvent,
+    'CyberAttack': CyberAttackEvent,
+    'DiseaseOutbreak': DiseaseOutbreakEvent,
+    'Election': PositionEvent,
+    'EnvironmentalIssue': EnvironmentalIssueEvent,
+    'MalwareAnalysis': MalwareAnalysisEvent,
+    'ManMadeDisaster': ManMadeDisasterEvent,
+    'MilitaryManeuver': MilitaryManeuverEvent,
+    'NaturalDisaster': NaturalDisasterEvent,
+    'NuclearMaterialTransaction': NuclearMaterialTransactionEvent,
+    'PersonThreat': PersonThreatEvent,
+    'PoliticalEvent': PositionEvent,
+    'PublicSafetyWarning': PositionEvent,
+    'RFEVEArmedAssault': PositionEvent,
+    'RFEVEProtest': ProtestEvent,
+    'TerrorIncident': PositionEvent,
+}
+
+
+class NoteEvent(RFBaseModel):
+    type_: Optional[str] = Field(alias='type', default=None)
+    attributes: Optional[Any] = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def validate_attribute(cls, values):
+        """Validate note event attributes."""
+        if not values.get('type') or not values.get('attributes'):
+            raise ValueError('Missing type or attributes from note event')
+
+        type_ = values['type']
+        validator = ATTRIBUTES_MAPPING.get(type_)
+        if not validator:
+            log = logging.getLogger(__name__)
+            log.warning(f'Unknown validator for Analyst Note with event type {type_}')
+            return {}
+
+        try:
+            attributes = validator.model_validate(values['attributes'])
+        except ValidationError as e:
+            log = logging.getLogger(__name__)
+            log.warning(f'Failed to validate note event of type {type_}. Error {e}')
+            log.warning(values)
+            return {}
+
+        return {'type': type_, 'attributes': attributes}
+
+
+class Attributes(RFBaseModel):
+    title: str
+    text: str
+    published: datetime
+    attachment: Optional[str] = None
+    events: Optional[list[NoteEvent]] = []
+    validated_on: Optional[datetime] = None
+    note_entities: Optional[list[IdNameTypeDescription]] = []
+    context_entities: Optional[list[IdNameTypeDescription]] = []
+    topic: Optional[Union[list[IdNameTypeDescription], IdNameTypeDescription]] = []
+    labels: Optional[list[IdNameTypeDescription]] = []
+    validation_urls: Optional[list[IdNameTypeDescription]] = []
+    diamond_model: Optional[list[DiamondModel]] = []
+    recommended_queries: Optional[list[Query]] = []
+    header_image: Optional[IdNameType] = None
+
+    @field_validator('events', mode='after')
+    @classmethod
+    def remove_empty_events(cls, values):
+        """Remove empty events when ``NoteEvent`` skip the validation."""
+        return [v for v in values if v.type_ and v.attributes]
+
+
+class PreviewAttributesIn(RFBaseModel):
+    title: str
+    text: str
+    note_entities: Optional[list[str]] = []
+    context_entities: Optional[list[str]] = []
+    topic: Union[list[str], str, None] = []
+    labels: Optional[list[str]] = []
+    validation_urls: Optional[list[str]] = []
+
+
+class PreviewAttributesOut(RFBaseModel):
+    title: str
+    text: str
+    note_entities: Optional[list[IdNameTypeDescription]] = []
+    context_entities: Optional[list[IdNameTypeDescription]] = []
+    topic: Optional[list[IdNameTypeDescription]] = []
+    labels: Optional[list[IdNameTypeDescription]] = []
+    validation_urls: Optional[list[IdNameTypeDescription]] = []
+
+
+class RequestAttachment(RFBaseModel):
+    content_type: str
+    encoding: str
+    content: str

psengine/analyst_notes/note.py

@@ -0,0 +1,149 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from functools import total_ordering
+from typing import Optional, Union
+
+from pydantic import Field
+
+from ..common_models import IdNameTypeDescription, RFBaseModel
+from ..constants import TIMESTAMP_STR
+from .constants import NOTES_PER_PAGE, URL_TO_PORTAL
+from .models import Attributes, PreviewAttributesIn, PreviewAttributesOut, RequestAttachment
+
+
+# AnalystNote also used by BaseEnrichedEntity
+@total_ordering
+class AnalystNote(RFBaseModel):
+    """Validate data received from ``/search``, ``/analystnote/{note}`` endpoints.
+
+    Methods:
+        __hash__:
+            Returns a hash value based on note ``id_``.
+
+        __eq__:
+            Checks equality between two AnalystNote instances based on ``id_`` and published time.
+
+        __gt__:
+            Defines a greater-than comparison between two AnalystNote instances based published time
+            and the ``id_``
+
+        __str__:
+            Returns a string representation of the AnalystNote instance with:
+            ``id_``, ``title``, and published timestamp.
+
+            .. code-block:: python
+
+                >>> print(analyst_note)
+                Analyst Note ID: 12345, Title: Cyber Vuln, Published: 2024-05-21 10:42:30AM
+
+    Total Ordering:
+        The ordering of AnalystNote instances is determined primarily by the published timestamp in
+        the attributes. If two instances have the same published timestamp, the note id is used as
+        a secondary criterion for ordering.
+    """
+
+    external_id: Optional[str] = None
+    source: IdNameTypeDescription
+    attributes: Attributes
+    id_: str = Field(alias='id')
+
+    def __hash__(self):
+        return hash((self.id_, self.attributes.published))
+
+    def __eq__(self, other: 'AnalystNote'):
+        return (self.id_, self.attributes.published) == (other.id_, other.attributes.published)
+
+    def __gt__(self, other: 'AnalystNote'):
+        return (self.attributes.published, self.id_) > (other.attributes.published, other.id_)
+
+    def __str__(self):
+        return (
+            f'Analyst Note ID: {self.id_}, Title: {self.attributes.title}, '
+            f'Published: {self.attributes.published.strftime(TIMESTAMP_STR)}'
+        )
+
+    @property
+    def detection_rule_type(self) -> Optional[str]:
+        """Returns the attachment type if present, else None. It checks for specific types like
+        'sigma rule', 'yara rule', and 'snort rule' in the topics of the note.
+        """
+        topics_type = ('sigma rule', 'yara rule', 'snort rule')
+
+        topics = (
+            {topic.name.lower() for topic in self.attributes.topic if topic.name}
+            if isinstance(self.attributes.topic, list)
+            else [self.attributes.topic.name]
+        )
+
+        return next(
+            (topic_type.split()[0] for topic_type in topics_type if topic_type in topics),
+            None,
+        )
+
+    @property
+    def portal_url(self) -> str:
+        """Get the link to portal."""
+        if self.id_.startswith('doc:'):
+            return URL_TO_PORTAL.format(self.id_)
+        return URL_TO_PORTAL.format(f'doc:{self.id_}')
+
+
+class AnalystNotePreviewIn(RFBaseModel):
+    """Validate data sent to ``/preview`` endpoint."""
+
+    attributes: PreviewAttributesIn
+    source: Optional[str]
+    tagged_text: bool = False
+    serialization: str = 'full'
+
+
+class AnalystNotePreviewOut(RFBaseModel):
+    """Validate data received from ``/preview`` endpoint."""
+
+    attributes: PreviewAttributesOut
+    source: IdNameTypeDescription
+
+
+class AnalystNotePublishIn(AnalystNotePreviewIn):
+    """Validate data sent to ``/publish`` endpoint."""
+
+    attributes: PreviewAttributesIn
+    source: Optional[str] = None
+    tagged_text: bool = False
+    serialization: str = 'full'
+    note_id: Optional[str] = None
+    resolve_entities: bool = True
+    attachment_content_details: Optional[RequestAttachment] = None
+
+
+class AnalystNotePublishOut(RFBaseModel):
+    """Validate data received from ``/publish`` endpoint."""
+
+    note_id: str
+
+
+class AnalystNoteSearchIn(RFBaseModel):
+    """Validate data sent to ``/search`` endpoint."""
+
+    published: Optional[str] = None
+    entity: Optional[str] = None
+    author: Optional[str] = None
+    title: Optional[str] = None
+    topic: Union[list[str], str, None] = []
+    label: Optional[str] = None
+    source: Optional[str] = None
+    serialization: str = None
+    tagged_text: bool = None
+    limit: int = NOTES_PER_PAGE
+    from_: Optional[str] = Field(alias='from', default=None)