psengine 2.0.4__py3-none-any.whl

psengine/entity_lists/constants.py

@@ -0,0 +1,19 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+IS_READY_INCREMENT = 5
+
+ADD_OP = 'add'
+REMOVE_OP = 'remove'
+UNCHANGED_NAME = 'unchanged'
+ERROR_NAME = 'error'

psengine/entity_lists/entity_list.py

@@ -0,0 +1,435 @@
+#################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+import time
+from datetime import datetime
+from functools import total_ordering
+from typing import Optional, Union
+
+from pydantic import ConfigDict, Field, validate_call
+
+from ..common_models import IdNameType, RFBaseModel
+from ..constants import TIMESTAMP_STR
+from ..endpoints import EP_LIST
+from ..entity_match import EntityMatchMgr, MatchApiError
+from ..helpers import debug_call
+from ..helpers.helpers import connection_exceptions
+from ..rf_client import RFClient
+from .constants import ADD_OP, ERROR_NAME, IS_READY_INCREMENT, REMOVE_OP, UNCHANGED_NAME
+from .errors import ListApiError
+from .models import (
+    AddEntityRequestModel,
+    ListEntityOperationResponse,
+    OwnerOrganisationDetails,
+    RemoveEntityRequestModel,
+)
+
+
+class ListInfoOut(RFBaseModel):
+    """Validate data received from ``/{listId}/info`` endpoint."""
+
+    id_: str = Field(alias='id')
+    name: str
+    type_: str = Field(alias='type')
+    created: datetime
+    updated: datetime
+    owner_organisation_details: OwnerOrganisationDetails = Field(
+        default_factory=OwnerOrganisationDetails
+    )
+    owner_id: str
+    owner_name: str
+    organisation_id: str
+    organisation_name: str
+
+
+class ListStatusOut(RFBaseModel):
+    """Validate data received from ``/{listId}/status`` endpoint."""
+
+    size: int
+    status: str
+
+
+@total_ordering
+class ListEntity(RFBaseModel):
+    """Validate data received from ``/{listId}/entities`` endpoint."""
+
+    entity: IdNameType
+    context: Optional[dict] = None
+    status: str
+    added: datetime
+
+    def __hash__(self):
+        return hash(self.entity.id_)
+
+    def __eq__(self, other: 'ListEntity'):
+        return self.entity.id_ == other.entity.id_
+
+    def __gt__(self, other: 'ListEntity'):
+        return (self.entity.name, self.added) > (other.entity.name, other.added)
+
+    def __str__(self):
+        return (
+            f'{self.entity.type_}: {self.entity.name}, added {self.added.strftime(TIMESTAMP_STR)}'
+        )
+
+
+class EntityList(RFBaseModel):
+    """Validate data received from ``/create`` endpoint."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    rf_client: RFClient = Field(exclude=True)
+    match_mgr: EntityMatchMgr = Field(exclude=True)
+    log: logging.Logger = Field(exclude=True, default=logging.getLogger(__name__))
+    id_: str = Field(alias='id')
+    name: str
+    type_: str = Field(alias='type')
+    created: datetime
+    updated: datetime
+    owner_id: str
+    owner_name: str
+    organisation_id: Optional[str] = None
+    organisation_name: Optional[str] = None
+    owner_organisation_details: OwnerOrganisationDetails = Field(
+        default_factory=OwnerOrganisationDetails
+    )
+
+    def __hash__(self):
+        return hash(self.id_)
+
+    def __eq__(self, other: 'EntityList'):
+        return self.id_ == other.id_
+
+    def __str__(self):
+        """String representation of the list.
+
+        Returns:
+            str: list data with standard info + entities
+        """
+
+        def format_date(date):
+            return date.strftime(TIMESTAMP_STR)
+
+        def format_field(name, value):
+            return f"{name}: {value or 'None'}"
+
+        main_fields = [
+            format_field('id', self.id_),
+            format_field('name', self.name),
+            format_field('type', self.type_),
+            format_field('created', format_date(self.created)),
+            format_field('last updated', format_date(self.updated)),
+            format_field('owner id', self.owner_id),
+            format_field('owner name', self.owner_name),
+            format_field('organisation id', self.organisation_id),
+            format_field('organisation name', self.organisation_name),
+        ]
+
+        org_details = self.owner_organisation_details
+        org_fields = [
+            format_field('owner id', org_details.owner_id),
+            format_field('owner name', org_details.owner_name),
+            format_field('enterprise id', org_details.enterprise_id),
+            format_field('enterprise name', org_details.enterprise_name),
+        ]
+
+        sub_orgs = org_details.organisations
+        if sub_orgs:
+            sub_org_str = '\n    '.join(
+                f'organisation id: {org.organisation_id}\n'
+                f'    organisation name: {org.organisation_name}'
+                for org in sub_orgs
+            )
+            org_fields.append(f'sub-organisations:\n    {sub_org_str}')
+        else:
+            org_fields.append('sub-organisations: None')
+
+        return (
+            '\n'.join(main_fields) + '\nowner organisation details:\n  ' + '\n  '.join(org_fields)
+        )
+
+    @debug_call
+    @validate_call
+    def add(
+        self, entity: Union[str, tuple[str, str]], context: dict = None
+    ) -> ListEntityOperationResponse:
+        """Add entity to list.
+
+        Endpoint:
+            ``list/{id}/entity/add``
+
+        Args:
+            entity (str, tuple): ID or (name, type) tuple of entity to add
+            context (dict, optional): context object for entity. Defaults to None
+
+        Raises:
+            ListApiError: if connection error occurs.
+
+        Returns:
+            ListEntityOperationResponse: list/{id}/entity/add response
+        """
+        return self._list_op(entity, ADD_OP, context=context or {})
+
+    @debug_call
+    @validate_call
+    def remove(self, entity: Union[str, tuple[str, str]]) -> ListEntityOperationResponse:
+        """Remove entity from list.
+
+        Endpoint:
+            ``list/{id}/entity/remove``
+
+        Args:
+            entity (str, tuple): ID or (name, type) tuple of entity to remove
+
+        Raises:
+            ListApiError: if connection error occurs.
+
+        Returns:
+            ListEntityOperationResponse: list/{id}/entity/remove response
+        """
+        return self._list_op(entity, REMOVE_OP)
+
+    @debug_call
+    @validate_call
+    def bulk_add(self, entities: list[Union[str, tuple[str, str]]]) -> dict:
+        """Bulk add entities to list.
+
+        Adds entities 1 at a time due to List API requirement. Logs progress every 10%.
+
+        Endpoint:
+            ``list/{id}/entity/add``
+
+        Args:
+            entities (list[Union[str, tuple[str, str]]]): list of entity string IDs or
+            entity (name, type) tuples to add
+
+        Raises:
+            ValidationError if any supplied parameter is of incorrect type.
+            ValueError: if wrong operation is supplied
+            ListApiError: if connection error occurs
+
+        Returns:
+            dict: results JSON with added, unchanged, error keys containing lists of entities
+        """
+        result = self._bulk_op(entities, ADD_OP)
+        status = self.status()
+        while status.status != 'ready':
+            self.log.info(f"Awaiting list 'ready' status, current status '{status.status}'")
+            status = self.status()
+            time.sleep(IS_READY_INCREMENT)
+
+        return result
+
+    @debug_call
+    @validate_call
+    def bulk_remove(self, entities: list[Union[str, tuple[str, str]]]) -> dict:
+        """Bulk remove entities from list.
+
+        Removes entities 1 at a time due to List API requirement. Logs progress every 10%.
+
+        Endpoint:
+            ``list/{id}/entity/remove``
+
+        Args:
+            entities (list): list of entity string IDs or entity (name, type) tuples to remove
+
+        Raises:
+            ValidationError if any supplied parameter is of incorrect type.
+            ValueError: if wrong operation is supplied
+            ListApiError: if connection error occurs
+
+        Returns:
+            dict: results JSON with removed, unchanged, error keys containing lists of entities
+        """
+        result = self._bulk_op(entities, REMOVE_OP)
+        status = self.status()
+        while status.status != 'ready':
+            self.log.info(f"Awaiting list 'ready' status, current status '{status.status}'")
+            status = self.status()
+            time.sleep(IS_READY_INCREMENT)
+
+        return result
+
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def entities(self) -> list[ListEntity]:
+        """Get entities for list.
+
+        Endpoint:
+            ``list/{id}/entities``
+
+        Raises:
+            ListApiError: if connection error occurs.
+
+        Returns:
+            list[ListEntity]: list/{id}/entities JSON response
+        """
+        url = EP_LIST + '/' + self.id_ + '/entities'
+        response = self.rf_client.request('get', url)
+        return [ListEntity.model_validate(entity) for entity in response.json()]
+
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def text_entries(self) -> list[str]:
+        """Get text entries for list.
+
+        Endpoint:
+            ``list/{id}/textEntries``
+
+        Raises:
+            ListApiError: if connection error occurs.
+
+        Returns:
+            list[str]: list/{id}/textEntries JSON response
+        """
+        url = EP_LIST + '/' + self.id_ + '/textEntries'
+        return self.rf_client.request('get', url).json()
+
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def status(self) -> ListStatusOut:
+        """Get status information about list.
+
+        Endpoint:
+            ``list/{id}/status``
+
+        Raises:
+            ListApiError: if connection error occurs
+
+        Returns:
+            ListStatusOut: list/{id}/status response
+        """
+        self.log.debug(f"Getting list status for '{self.name}'")
+        url = EP_LIST + f'/{self.id_}/status'
+        response = self.rf_client.request('get', url)
+        validated_status = ListStatusOut.model_validate(response.json())
+        self.log.debug(
+            f"List '{self.name}' status: {validated_status.status}, entities: {validated_status.size}"  # noqa: E501
+        )
+
+        return validated_status
+
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def info(self) -> ListInfoOut:
+        """Get info for list.
+
+        Endpoint:
+            ``list/{id}/info``
+
+        Raises:
+            ListApiError: if connection error occurs
+
+        Returns:
+            ListInfoOut: list/{id}/info response
+        """
+        self.log.debug(f"Getting list status for '{self.name}'")
+        url = EP_LIST + f'/{self.id_}/info'
+        response = self.rf_client.request('get', url)
+        validated_info = ListInfoOut.model_validate(response.json())
+
+        return validated_info
+
+    @debug_call
+    def _bulk_op(self, entities: list[Union[str, tuple[str, str]]], operation: str) -> dict:
+        """Bulk add or remove entities from list.
+
+        List API requires that entities are added one at a time. Logs progress every 10%
+
+        Args:
+            entities (list): list of entity string IDs or (name, type) tuples to add
+            operation (str): the operation to perform on the list. Can be 'added' or 'removed'.
+
+        Raises:
+            ValueError: if wrong operation is supplied
+            ListApiError: if connection error occurs
+
+        Returns:
+            dict: results JSON with added, unchanged, error keys containing lists of entities added
+        """
+        if operation == ADD_OP:
+            op_func = self.add
+            op_name = 'added'
+        elif operation == REMOVE_OP:
+            op_func = self.remove
+            op_name = 'removed'
+        else:
+            raise ValueError(f"Operation must be either '{ADD_OP}' or '{REMOVE_OP}'")
+        result = {op_name: [], UNCHANGED_NAME: [], ERROR_NAME: []}
+        total = len(entities)
+        step = 10
+        for idx, entity in enumerate(entities):
+            try:
+                if isinstance(entity, str):
+                    entity_id = entity
+                else:  # entity is tuple
+                    entity_id = self.match_mgr.resolve_entity_id(entity[0], entity_type=entity[1])
+                    if not entity_id.is_found:
+                        result[ERROR_NAME].append({'message': entity_id.content, 'id': entity})
+                        continue
+                    else:
+                        entity_id = entity_id.content.id_
+                response = op_func(entity)
+                if response.result == op_name:
+                    result[op_name].append(entity_id)
+                elif response.result == UNCHANGED_NAME:
+                    result[UNCHANGED_NAME].append(entity_id)
+            except (TypeError, ListApiError, MatchApiError) as err:
+                result[ERROR_NAME].append({'message': str(err), 'id': entity})
+            if ((idx + 1) / total) * 100 >= step:
+                self.log.info(f'{op_name.capitalize()} {step}% of entities')
+                step += 10
+
+        return result
+
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def _list_op(
+        self, entity: Union[str, tuple[str, str]], op_name: str, context: dict = None
+    ) -> ListEntityOperationResponse:
+        """Add or remove an entity from list.
+
+        Args:
+            entity (str, tuple): ID or (name, type) tuple of entity to add
+            op_name (str): operation to perform. Either 'added' or 'removed'
+            context (dict, optional): context object for entity. Defaults to {}
+
+        Raises:
+            ListApiError: if connection error occurs
+
+        Returns:
+            ListEntityOperationResponse: list/{id}/entity/[add|remove] response
+        """
+        if isinstance(entity, str):
+            resolved_entity_id = entity
+        else:
+            resolved_entity = self.match_mgr.resolve_entity_id(entity[0], entity_type=entity[1])
+            if not resolved_entity.is_found:
+                return ListEntityOperationResponse(result=resolved_entity.content)
+            resolved_entity_id = resolved_entity.content.id_
+
+        url = EP_LIST + f'/{self.id_}/entity/' + op_name
+        request_body = {'entity': {'id': resolved_entity_id}}
+
+        if context:
+            request_body['context'] = context
+        if op_name == ADD_OP:
+            AddEntityRequestModel.model_validate(request_body)
+        else:
+            RemoveEntityRequestModel.model_validate(request_body)
+        response = self.rf_client.request('post', url, data=request_body)
+        validated_response = ListEntityOperationResponse.model_validate(response.json())
+        if validated_response.result != UNCHANGED_NAME:
+            self.log.debug(f'Entity {entity} {validated_response.result} to list {self.id_}')
+
+        return validated_response

psengine/entity_lists/entity_list_mgr.py

@@ -0,0 +1,185 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+from typing import Union
+
+from pydantic import validate_call
+
+from ..constants import DEFAULT_LIMIT
+from ..endpoints import EP_CREATE_LIST, EP_LIST, EP_SEARCH_LIST
+from ..entity_match import EntityMatchMgr
+from ..helpers import debug_call
+from ..helpers.helpers import connection_exceptions
+from ..rf_client import RFClient
+from .entity_list import EntityList
+from .errors import ListApiError, ListResolutionError
+
+
+class EntityListMgr:
+    """Manages requests for Recorded Future List API."""
+
+    def __init__(self, rf_token: str = None) -> None:
+        """Initialize EntityListMgr object.
+
+        Args:
+            rf_token (str, optional): Recorded Future API token. Defaults to None
+        """
+        self.log = logging.getLogger(__name__)
+        self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient()
+        self.match_mgr = EntityMatchMgr(rf_token=rf_token) if rf_token else EntityMatchMgr()
+
+    @debug_call
+    @validate_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def fetch(self, list_: Union[str, tuple[str, str]]) -> EntityList:
+        """Gets a list by its ID. Use this function for list info response.
+
+        Endpoint:
+            ``list/{list_id}/info``
+
+        Args:
+            list_ (str, tuple): list string ID or tuple of (name, type)
+
+        Raises:
+            ValidationError if any supplied parameter is of incorrect type.
+            ListResolutionError: when ``list_`` is a tuple and name matches 0 or multiple entities
+            ListApiError: if connection error occurs.
+
+        Returns:
+            RFList: RFList object for list ID
+        """
+        resolved_id = self._resolve_list_id(list_)
+        self.log.info(f'Getting list with ID: {resolved_id}')
+        url = EP_LIST + f'/{resolved_id}/info'
+        response = self.rf_client.request('get', url)
+        list_info_data = response.json()
+        self.log.debug("Found list ID '{}'".format(list_info_data['id']))
+        self.log.debug('  Type: {}'.format(list_info_data['type']))
+        self.log.debug('  Created: {}'.format(list_info_data['created']))
+        self.log.debug('  Updated: {}'.format(list_info_data['updated']))
+
+        return EntityList(rf_client=self.rf_client, match_mgr=self.match_mgr, **list_info_data)
+
+    @debug_call
+    @validate_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def create(self, list_name: str, list_type: str = 'entity') -> EntityList:
+        """Create list.
+
+        Endpoint:
+            ``list/create``
+
+        Args:
+            list_name (str): list name to use for new list
+            list_type (str, optional): list type. Defaults to "entity"
+
+            Supported list types are available on the support page for List API:
+            https://support.recordedfuture.com/hc/en-us/articles/360058691913-List-API
+
+        Raises:
+            ValidationError if any supplied parameter is of incorrect type.
+            ListApiError: if connection error occurs.
+
+        Returns:
+            EntityList: EntityList object for new list
+        """
+        self.log.debug(f"Creating list '{list_name}'")
+        request_body = {'name': list_name, 'type': list_type}
+        response = self.rf_client.request('post', EP_CREATE_LIST, data=request_body)
+        list_create_data = response.json()
+        self.log.debug(f"List '{list_name}' created")
+        self.log.debug('  ID: {}'.format(list_create_data['id']))
+        self.log.debug('  Type: {}'.format(list_create_data['type']))
+
+        return EntityList(rf_client=self.rf_client, match_mgr=self.match_mgr, **list_create_data)
+
+    @debug_call
+    @validate_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
+    def search(
+        self, list_name: str = None, list_type: str = None, max_results: int = DEFAULT_LIMIT
+    ) -> list[EntityList]:
+        """Search lists.
+
+        Endpoint:
+            ``list/search``
+
+        Args:
+            list_name (str): list name to search
+            list_type (str, optional): list type. Defaults to None, ignored when None
+            max_results (int, optional): maximum total number of lists to return
+
+        Raises:
+            ValidationError if any supplied parameter is of incorrect type.
+            ListApiError: if list API call fails
+
+        Returns:
+            list: EntityList objects from list/search
+        """
+        request_body = {}
+        request_body['limit'] = max_results
+        if list_name:
+            request_body['name'] = list_name
+        if list_type:
+            request_body['type'] = list_type
+        self.log.info(f'Searching list API with parameters: {request_body}')
+        response = self.rf_client.request('post', EP_SEARCH_LIST, data=request_body)
+        list_search_data = response.json()
+        self.log.info(
+            'Found {} matching {}'.format(
+                len(list_search_data), 'lists' if len(list_search_data) != 1 else 'list'
+            )
+        )
+
+        return [
+            EntityList(rf_client=self.rf_client, match_mgr=self.match_mgr, **list_)
+            for list_ in list_search_data
+        ]
+
+    @debug_call
+    def _resolve_list_id(self, list_: Union[str, tuple[str, str]]) -> str:
+        """Resolves a list name to a list ID.
+
+        Args:
+            list_ (str, tuple): list string ID or (name, type) tuple
+
+        Raises:
+            ListResolutionError: when a list name matches none or multiple entities
+
+        Returns:
+            str: list ID
+        """
+        if isinstance(list_, str):
+            resolved_id = list_
+        else:
+            list_name, list_type = list_
+            self.log.info(f"Resolving ID for list '{list_name}' with type '{list_type}'")
+            matches = self.search(list_name, list_type)
+            if len(matches) == 0:
+                message = f"No match found for string '{list_name}'"
+                raise ListResolutionError(message)
+            elif len(matches) > 1:
+                exact_count = 0
+                resolved_id = None
+                for match in matches:
+                    if match.name == list_name:
+                        resolved_id = match.id_
+                        exact_count += 1
+                if (not resolved_id) or exact_count > 1:
+                    message = f"Multiple matches found for string '{list_name}'"
+                    raise ListResolutionError(message)
+            else:
+                resolved_id = matches[0].id_
+
+        return resolved_id

psengine/entity_lists/errors.py

@@ -0,0 +1,26 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from ..errors import RecordedFutureError
+
+
+class ListApiError(RecordedFutureError):
+    """Error raised when an exception occurs performing a list API operation."""
+
+
+class ListStateError(RecordedFutureError):
+    """Error raised when list state is invalid."""
+
+
+class ListResolutionError(RecordedFutureError):
+    """Error raised when list resolution fails."""