psengine 2.0.4__py3-none-any.whl

psengine/enrich/models/lookup.py

@@ -0,0 +1,271 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from datetime import datetime
+from typing import Optional, Union
+
+from pydantic import Field
+
+from ...common_models import IdNameType, IdNameTypeDescription, RFBaseModel
+
+
+###########################################################
+# Enterprise Lists
+###########################################################
+class EnterpriseList(RFBaseModel):
+    added: Optional[datetime]
+    list_: IdNameTypeDescription = Field(alias='list')
+
+
+###########################################################
+class RiskyCIDRPIP(RFBaseModel):
+    score: int
+    ip: IdNameType
+
+
+class AIInsights(RFBaseModel):
+    comment: Optional[str] = None
+    text: Optional[str] = None
+    number_of_references: Optional[int] = Field(alias='numberOfReferences', default=None)
+
+
+class EvidenceDetails(RFBaseModel):
+    mitigation_string: str = Field(alias='mitigationString')
+    evidence_string: str = Field(alias='evidenceString')
+    rule: str
+    criticality: int
+    timestamp: datetime
+    criticality_label: str = Field(alias='criticalityLabel')
+
+
+class EntityRisk(RFBaseModel):
+    criticality_label: str = Field(alias='criticalityLabel')
+    risk_string: str = Field(alias='riskString')
+    rules: int
+    criticality: int
+    risk_summary: str = Field(alias='riskSummary')
+    score: int
+    evidence_details: list[EvidenceDetails] = Field(alias='evidenceDetails')
+
+
+class Sighting(RFBaseModel):
+    source: str
+    url: str
+    published: datetime
+    fragment: str
+    title: str
+    type_: str = Field(alias='type')
+
+
+class RiskMappingCategory(RFBaseModel):
+    framework: str
+    name: str
+
+
+class RiskMapping(RFBaseModel):
+    rule: str
+    categories: Optional[list[RiskMappingCategory]] = None
+
+
+class RelatedEntity(RFBaseModel):
+    count: int
+    entity: IdNameTypeDescription
+
+
+class RelatedEntities(RFBaseModel):
+    entities: list[RelatedEntity]
+    type_: str = Field(alias='type')
+
+
+class GeoLocation(RFBaseModel):
+    continent: Optional[str] = None
+    country: Optional[str] = None
+    city: Optional[str] = None
+
+
+class IPLocation(RFBaseModel):
+    organization: Optional[str]
+    cidr: IdNameType
+    location: GeoLocation
+    asn: Optional[str] = None
+
+
+class Timestamps(RFBaseModel):
+    last_seen: datetime = Field(alias='lastSeen')
+    first_seen: datetime = Field(alias='firstSeen')
+
+
+class ReferenceCount(RFBaseModel):
+    date: datetime
+    count: int
+
+
+class Metric(RFBaseModel):
+    type_: str = Field(alias='type')
+    value: Union[int, float]
+
+
+###########################################################
+# Links
+###########################################################
+class LinksCounts(RFBaseModel):
+    count: int
+    type_: IdNameTypeDescription = Field(alias='type')
+
+
+class LinksList(RFBaseModel):
+    entities: list[IdNameTypeDescription]
+    total_count: int
+    type_: IdNameTypeDescription = Field(alias='type')
+
+
+class SectionHits(RFBaseModel):
+    section_id: IdNameType
+    total_count: int
+    lists: Optional[list[LinksList]] = None
+
+
+class Hits(RFBaseModel):
+    sections: list[SectionHits]
+    start_date: datetime
+    stop_date: datetime
+    total_count: int
+    sample_reference_ids: list[str]
+    counts: list[LinksCounts]
+    event_count: int
+
+
+class MethodAggregate(RFBaseModel):
+    count: int
+    type_: str = Field(alias='type')
+
+
+class Links(RFBaseModel):
+    hits: list[Hits]
+    method_aggregates: list[MethodAggregate]
+    counts: list[LinksCounts]
+
+
+###########################################################
+# Linked Malware
+###########################################################
+class LinkedMalware(RFBaseModel):
+    entities: list[IdNameType]
+    total_count: int
+
+
+###########################################################
+# CVSS
+###########################################################
+class CVSS(RFBaseModel):
+    access_vector: Optional[str] = Field(alias='accessVector', default=None)
+    last_modified: Optional[datetime] = Field(alias='lastModified', default=None)
+    published: Optional[datetime] = None
+    score: Optional[float] = None
+    availability: Optional[str] = None
+    authentication: Optional[str] = None
+    access_complexity: Optional[str] = Field(alias='accessComplexity', default=None)
+    integrity: Optional[str] = None
+    confidentiality: Optional[str] = None
+    version: Optional[str] = None
+
+
+class CVSSRating(RFBaseModel):
+    score: float
+    modified: datetime
+    version: str
+    type_: str = Field(alias='type')
+    created: datetime
+
+
+class CVSSV3(RFBaseModel):
+    scope: Optional[str] = None
+    exploitability_score: Optional[float] = Field(alias='exploitabilityScore', default=None)
+    modified: Optional[datetime] = None
+    base_severity: Optional[str] = Field(alias='baseSeverity', default=None)
+    base_score: Optional[float] = Field(alias='baseScore', default=None)
+    privileges_required: Optional[str] = Field(alias='privilegesRequired', default=None)
+    user_interaction: Optional[str] = Field(alias='userInteraction', default=None)
+    impact_score: Optional[float] = Field(alias='impactScore', default=None)
+    attack_vector: Optional[str] = Field(alias='attackVector', default=None)
+    integrity_impact: Optional[str] = Field(alias='integrityImpact', default=None)
+    confidentiality_impact: Optional[str] = Field(alias='confidentialityImpact', default=None)
+    vector_string: Optional[str] = Field(alias='vectorString', default=None)
+    version: Optional[str] = None
+    attack_complexity: Optional[str] = Field(alias='attackComplexity', default=None)
+    created: Optional[datetime] = None
+    availability_impact: Optional[str] = Field(alias='availabilityImpact', default=None)
+
+
+###########################################################
+# Raw Risk
+###########################################################
+class RawRisk(RFBaseModel):
+    rule: str
+    timestamp: datetime
+
+
+###########################################################
+# DNS Port Cert
+###########################################################
+class Validity(RFBaseModel):
+    valid_from: datetime = Field(alias='validFrom')
+    valid_to: datetime = Field(alias='validTo')
+
+
+class Issuer(RFBaseModel):
+    organization: Optional[str] = None
+    location: Optional[str] = None
+
+
+class Certificate(RFBaseModel):
+    subject: Optional[str] = None
+    validity: Validity
+    issuer: Issuer
+    seen_on_port: list[int] = Field(alias='seenOnPort')
+
+
+class ForwardDNS(RFBaseModel):
+    hostname: Optional[str] = None
+    last_seen: Union[datetime, None] = Field(alias='lastSeen')
+    first_seen: Union[datetime, None] = Field(alias='firstSeen')
+
+
+class DNS(RFBaseModel):
+    forward_dns: list[ForwardDNS] = Field(alias='forwardDns')
+    reverse_dns: Optional[str] = Field(alias='reverseDns', default=None)
+
+
+class Port(RFBaseModel):
+    name: str
+    version: Union[str, None]
+    port: int
+    extra_info: Union[str, None] = Field(alias='extraInfo')
+    protocol: str
+    product: Union[str, None]
+
+
+class DnsPortCert(RFBaseModel):
+    certificates: Optional[list[Certificate]] = None
+    dns: Optional[DNS] = None
+    ports: Optional[list[Port]] = None
+
+
+###########################################################
+# NVD
+###########################################################
+
+
+class NvdReference(RFBaseModel):
+    url: str
+    tags: list[str]

psengine/enrich/models/soar.py

@@ -0,0 +1,138 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import Field, model_validator
+
+from ...common_models import RFBaseModel
+
+
+class ScoreCount(RFBaseModel):
+    count: Optional[int] = None
+    max_count: int = Field(alias='maxCount')
+
+
+class ScoreRule(RFBaseModel):
+    score: int
+    rule: ScoreCount
+
+
+class Public(ScoreRule):
+    summary: list
+    most_critical_rule: str = Field(alias='mostCriticalRule')
+
+
+class Evidence(RFBaseModel):
+    count: int
+    timestamp: datetime
+    description: str
+    rule: str
+    # TODO - temp fix until API team fixes/confirms behaviour of sightings
+    sightings: int = 0
+    mitigation: str
+    level: int
+    type_: str = Field(alias='type')
+
+
+class RiskRule(ScoreCount):
+    score: Optional[int] = None
+    summary: list
+    most_critical: str = Field(alias='mostCritical')
+    evidence: Optional[list[Evidence]] = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def evidence_transform(cls, data: dict) -> dict:
+        """Transforms the evidence field into a list of dicts, each with a 'type' key.
+
+        From:
+
+        .. code-block::
+
+            "evidence": {
+                "recentValidatedCnc": {
+                    "count": 1,
+                    "timestamp": "2024-03-25T07:18:35.000Z",
+                    "description": "xyz",
+                    "rule": "Validated C&C Server",
+                    "sightings": 41,
+                    "mitigation": "",
+                    "level": 4
+                },
+                "recentSuspectedCnc": {
+                    "count": 1,
+                    "timestamp": "2024-03-24T16:05:31.634Z",
+                    "description": "xyz",
+                    "rule": "Recent Suspected C&C Server",
+                    "sightings": 5,
+                    "mitigation": "",
+                    "level": 2
+                }
+            }
+
+        To:
+
+        .. code-block::
+
+            "evidence": [
+                {
+                    "count": 1,
+                    "timestamp": "2023-12-11T19:25:25.892000Z",
+                    "description": "xyz",
+                    "sightings": 1,
+                    "mitigation": "",
+                    "level": 3,
+                    "type": "recentReportedCnc"
+                },
+                {
+                    "count": 2,
+                    "timestamp": "2023-12-25T22:09:55.398000Z",
+                    "description": "xyz",
+                    "rule": "Historical Suspected C&C Server",
+                    "sightings": 2,
+                    "mitigation": "",
+                    "level": 1,
+                    "type": "suspectedCnc"
+                }
+            ]
+
+        Args:
+            data (dict): The data to be validated.
+
+        Returns:
+            dict: The transformed data.
+        """
+        if 'evidence' not in data:
+            return data
+
+        evidence = data.pop('evidence')
+
+        data['evidence'] = [{**ev, 'type': key} for key, ev in evidence.items()]
+
+        return data
+
+
+class Context(RFBaseModel):
+    phishing: Optional[ScoreRule] = None
+    public: Optional[Public] = None
+    c2: Optional[ScoreRule] = None
+    malware: Optional[ScoreRule] = None
+
+
+class Risk(RFBaseModel):
+    score: int
+    level: int
+    context: Context
+    rule: RiskRule

psengine/enrich/soar.py

@@ -0,0 +1,89 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from functools import total_ordering
+from typing import Optional
+
+from pydantic import Field
+
+from ..common_models import IdNameTypeDescription, RFBaseModel
+from .models.soar import Risk
+
+
+@total_ordering
+class SOAREnrichedEntity(RFBaseModel):
+    """Model used for validating returned data from SOAR enrichment endpoint for bulk enrichment.
+
+    Methods:
+        __hash__:
+            Returns a hash value based on the entity ``id_`` and the risk score.
+
+        __eq__:
+            Checks equality between two ``SOAREnrichedEntity`` instances based on their entity name
+            and the risk score.
+
+        __gt__:
+            Defines a greater-than comparison between two ``SOAREnrichedEntity`` instances based on
+            their risk score and entity name.
+
+        __str__:
+            Returns a string representation of the ``SOAREnrichedEntity`` instance with:
+            enriched entity name, risk score, and most critical rule.
+
+            .. code-block:: python
+
+                >>> print(soar_enriched_entity)
+                Enriched Entity: 1.1.1.1, Risk Score: 95, Most Critical Rule: C&C Server
+
+    Total Ordering:
+        The ordering of ``SOAREnrichedEntity`` instances is determined primarily by the risk score.
+        If two instances have the same risk score, their entity name is used as a secondary
+        criterion for ordering.
+    """
+
+    risk: Risk
+    entity: IdNameTypeDescription
+
+    def __hash__(self):
+        return hash((self.entity.id_, self.risk.score))
+
+    def __eq__(self, other: 'SOAREnrichedEntity'):
+        return (self.entity.name, self.risk.score) == (other.entity.name, other.risk.score)
+
+    def __gt__(self, other: 'SOAREnrichedEntity'):
+        return (self.risk.score, self.entity.name) > (other.risk.score, other.entity.name)
+
+    def __str__(self):
+        return (
+            f'Enriched Entity: {self.entity.name}, Risk Score: {self.risk.score}, '
+            f'Most Critical Rule: {self.risk.rule.most_critical}'
+        )
+
+
+class SOAREnrichIn(RFBaseModel):
+    """Model used to validate payload sent to SOAR enrichment endpoint."""
+
+    ip: Optional[list[str]] = None
+    domain: Optional[list[str]] = None
+    url: Optional[list[str]] = None
+    hash_: Optional[list[str]] = Field(alias='hash', default=None)
+    vulnerability: Optional[list[str]] = None
+    companybydomain: Optional[list[str]] = None
+
+
+class SOAREnrichOut(RFBaseModel):
+    """Model used for collecting all the data returned in a SOAR call."""
+
+    entity: str
+    is_enriched: bool
+    content: Optional[SOAREnrichedEntity] = None

psengine/enrich/soar_mgr.py

@@ -0,0 +1,176 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+from itertools import chain
+from typing import Optional
+
+from pydantic import validate_call
+
+from ..endpoints import EP_SOAR_ENRICHMENT
+from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call
+from ..rf_client import RFClient
+from .constants import SOAR_POST_ROWS
+from .errors import EnrichmentSoarError
+from .soar import SOAREnrichedEntity, SOAREnrichIn, SOAREnrichOut
+
+
+class SoarMgr:
+    """Perform SOAR enrichment of entities."""
+
+    def __init__(self, rf_token: str = None):
+        """Initializes the SoarMgr object.
+
+        Args:
+            rf_token (str, optional): Recorded Future API token. Defaults to None
+        """
+        self.log = logging.getLogger(__name__)
+        self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient()
+
+    @validate_call
+    @debug_call
+    def soar(
+        self,
+        ip: Optional[list[str]] = None,
+        domain: Optional[list[str]] = None,
+        hash_: Optional[list[str]] = None,
+        vulnerability: Optional[list[str]] = None,
+        url: Optional[list[str]] = None,
+        companybydomain: Optional[list[str]] = None,
+        max_workers: Optional[int] = 0,
+    ) -> list[SOAREnrichOut]:
+        """Enrich multiple types of IOCs via the SOAR API.
+
+        Endpoint:
+            ``v2/soar/enrichment``
+
+        This method allows for batch processing of various IOC types, including IP addresses,
+        domains, file hashes, vulnerabilities, URLs, and company domains. It utilizes either
+        multi-threading or sequential processing based on the `max_workers` parameter.
+
+        Examples:
+            Simple SOAR enrichment:
+
+            .. code-block:: python
+                :linenos:
+
+                mgr.soar(ip=['1.1.1.1'])
+
+            Multithreaded example:
+
+            .. code-block:: python
+                :linenos:
+
+                mgr.soar(ip=['1.1.1.1'], max_workers=10)
+
+            To write an enriched object to file:
+
+            .. code-block:: python
+                :linenos:
+
+                from pathlib import Path
+                from json import dump
+
+                mgr = SoarMgr()
+                OUTPUT_DIR = Path('your' / 'path')
+                OUTPUT_DIR.mkdir(exists_ok=True)
+                data = mgr.soar(ip=['1.1.1.1', '8.8.8.8'])
+                for ip in data:
+                    (OUTPUT_DIR / f'{ip.entity}.json').write_text(dumps(ip.json(), indent=2))
+
+        Args:
+            ip (List[str], optional): List of IP addresses to enrich.
+            domain (List[str], optional): List of domains to enrich.
+            hash_ (List[str], optional): List of file hashes to enrich.
+            vulnerability (List[str], optional): List of vulnerabilities to enrich.
+            url (List[str], optional): List of URLs to enrich.
+            companybydomain (List[str], optional): List of company domains to enrich.
+            max_workers (int, optional): number of workers to multithread requests.
+
+        Returns:
+            List[SOAREnrichOut]: A list of enriched data for the provided IOCs.
+
+        Raises:
+            ValueError: If no parameters are provided or all provided lists are empty.
+            ValidationError If the arguments have incorrect types or if the data returned
+                                    by the API is malformed.
+            EnrichmentSoarError: If an HTTP or JSON decoding error occurs during enrichment.
+        """
+        iocs = {
+            'ip': ip,
+            'domain': domain,
+            'hash': hash_,
+            'vulnerability': vulnerability,
+            'url': url,
+            'companybydomain': companybydomain,
+        }
+        iocs = {k: v for k, v in iocs.items() if v}
+        if not iocs:
+            raise ValueError('At least one parameter must be used')
+
+        results = []
+        if max_workers:
+            results.append(
+                chain.from_iterable(
+                    MultiThreadingHelper.multithread_it(
+                        max_workers,
+                        self._fetch_data,
+                        iterator=self._batched_cross_entity(iocs, SOAR_POST_ROWS),
+                    )
+                )
+            )
+        else:
+            results = [
+                self._fetch_data(batched_iocs)
+                for batched_iocs in self._batched_cross_entity(iocs, SOAR_POST_ROWS)
+            ]
+
+        return list(chain.from_iterable(results))
+
+    def _batched_cross_entity(self, iocs, batch_size):
+        """Batches the SOAR data in dict of maximum SOAR_POST_ROWS elements in total.
+        It always return a list of SoarRequest compatible object as dict.
+        """
+        batches = []
+        current_batch = {k: [] for k in iocs}
+        current_count = 0
+
+        for k, vals in iocs.items():
+            for v in vals:
+                if current_count >= batch_size:
+                    batches.append({k: v for k, v in current_batch.items() if v})
+                    current_batch = {k: [] for k in iocs}
+                    current_count = 0
+
+                current_batch[k].append(v)
+                current_count += 1
+
+        batches.append({k: v for k, v in current_batch.items() if v})
+        return batches
+
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=EnrichmentSoarError)
+    @debug_call
+    def _fetch_data(self, data: dict) -> list[SOAREnrichOut]:
+        """Perform soar post request and raises EnrichementSoarError if something goes wrong."""
+        data = SOAREnrichIn.model_validate(data)
+        res = self.rf_client.request('post', EP_SOAR_ENRICHMENT, data=data.json()).json()['data'][
+            'results'
+        ]
+        result = []
+
+        for d in res:
+            content = SOAREnrichedEntity.model_validate(d)
+            entity = content.entity.name
+            result.append(SOAREnrichOut(entity=entity, is_enriched=True, content=content))
+
+        return result

psengine/entity_lists/__init__.py

@@ -0,0 +1,16 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from .entity_list import EntityList, ListEntity, ListInfoOut, ListStatusOut
+from .entity_list_mgr import EntityListMgr
+from .errors import ListApiError, ListResolutionError, ListStateError