psengine 2.12.0__tar.gz → 2.13.0__tar.gz

{psengine-2.12.0 → psengine-2.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: psengine
-Version: 2.12.0
+Version: 2.13.0
 Summary: psengine is a simple, yet elegant, library for rapid development of integrations with Recorded Future.
 Keywords: API,Recorded Future,Cyber Security Engineering,Threat Intelligence
 Author: Moise Medici, Patrick Kinsella, Ernest Bartosevic

{psengine-2.12.0 → psengine-2.13.0}/psengine/collective_insights/__init__.py

@@ -13,6 +13,9 @@
 
 from .collective_insights import CollectiveInsights
 from .constants import (
+    ATOP_USE_CASE_DETECTION,
+    ATOP_USE_CASE_HUNTING,
+    ATOP_USE_CASE_PREVENTION,
     DETECTION_SUB_TYPE_SIGMA,
     DETECTION_SUB_TYPE_SNORT,
     DETECTION_SUB_TYPE_YARA,
@@ -24,6 +27,10 @@ from .constants import (
     ENTITY_IP,
     ENTITY_URL,
     ENTITY_VULNERABILITY,
+    SUBMISSION_METHOD_API,
+    SUBMISSION_METHOD_INTEGRATION,
+    SUBMISSION_METHOD_SANDBOX,
 )
-from .errors import CollectiveInsightsError
-from .insight import Insight, InsightsIn, InsightsOut
+from .errors import CollectiveInsightsError, CollectiveInsightsSearchError
+from .insight import Insight, InsightsIn, InsightsOut, SearchIn
+from .models import SearchEntry, SearchFilters

{psengine-2.12.0 → psengine-2.13.0}/psengine/collective_insights/collective_insights.py

@@ -13,17 +13,20 @@
 
 import json
 import logging
-from typing import Annotated
+from datetime import datetime
+from typing import Annotated, Literal
 
-from pydantic import validate_call
+from pydantic import Field, validate_call
 from typing_extensions import Doc
 
-from ..endpoints import EP_COLLECTIVE_INSIGHTS_DETECTIONS
+from ..constants import DEFAULT_LIMIT
+from ..endpoints import EP_COLLECTIVE_INSIGHTS_DETECTIONS, EP_COLLECTIVE_INSIGHTS_SEARCH
 from ..helpers import connection_exceptions, debug_call
 from ..rf_client import RFClient
-from .constants import SUMMARY_DEFAULT
-from .errors import CollectiveInsightsError
-from .insight import Insight, InsightsIn, InsightsOut
+from .constants import SEARCH_MAX_LIMIT, SEARCH_PAGE_SIZE, SUMMARY_DEFAULT
+from .errors import CollectiveInsightsError, CollectiveInsightsSearchError
+from .insight import Insight, InsightsIn, InsightsOut, SearchIn
+from .models import Presence, SearchEntry
 
 
 class CollectiveInsights:
@@ -134,6 +137,131 @@ class CollectiveInsights:
 
         return InsightsIn.model_validate(response.json())
 
+    @validate_call
+    @debug_call
+    @connection_exceptions(ignore_status_code=[], exception_to_raise=CollectiveInsightsSearchError)
+    def search(
+        self,
+        indicator_type: Annotated[
+            list[str] | str | Presence | None,
+            Doc('IOC type filter (`ip`, `domain`, `hash`, `url`, `vulnerability`).'),
+        ] = None,
+        detection_type: Annotated[
+            list[str] | str | Presence | None,
+            Doc(
+                'Detection method filter (`correlation`, `playbook`, `detection_rule`, '
+                '`sandbox`, `threat_hunt`, `vulnerability_scan`).',
+            ),
+        ] = None,
+        submission_method: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Submission method filter (`api`, `integration`, `sandbox`).'),
+        ] = None,
+        organizations: Annotated[
+            list[str] | str | None, Doc('Filter by organization IDs (uhash).')
+        ] = None,
+        detection_rule_id: Annotated[
+            list[str] | str | Presence | None, Doc('Filter by associated detection rule IDs.')
+        ] = None,
+        detection_time_from: Annotated[
+            str | datetime | None, Doc('Start of the detection time range (inclusive).')
+        ] = None,
+        detection_time_to: Annotated[
+            str | datetime | None, Doc('End of the detection time range (inclusive).')
+        ] = None,
+        malware_id: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Filter by associated malware entity IDs.'),
+        ] = None,
+        mitre_code_id: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Filter by associated MITRE ATT&CK IDs (prefixed with `mitre:`).'),
+        ] = None,
+        threat_actor_id: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Filter by associated threat actor entity IDs.'),
+        ] = None,
+        atop_use_case: Annotated[
+            list[str] | str | Presence | None,
+            Doc(
+                'Filter by Autonomous Threat Operations use case '
+                '(`hunting`, `detection`, `prevention`).',
+            ),
+        ] = None,
+        atop_profile_id: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Filter by Autonomous Threat Operations profile ID.'),
+        ] = None,
+        atop_job_id: Annotated[
+            list[str] | str | Presence | None,
+            Doc('Filter by Autonomous Threat Operations job ID.'),
+        ] = None,
+        integration_type_id: Annotated[
+            list[str] | str | Presence | None, Doc('Filter by integration type entity IDs.')
+        ] = None,
+        indicator_risk_score: Annotated[
+            dict | Literal['present', 'absent'] | None,
+            Doc(
+                'Filter by indicator risk score at detection time. Pass `present`/`absent` '
+                'or a range dict such as `{"gte": 50, "lt": 90}`.',
+            ),
+        ] = None,
+        max_results: Annotated[int, Doc('Maximum number of events to return.')] = Field(
+            ge=1, default=DEFAULT_LIMIT
+        ),
+        page_size: Annotated[int, Doc('Number of events per page (max 1000).')] = Field(
+            ge=1, le=SEARCH_MAX_LIMIT, default=SEARCH_PAGE_SIZE
+        ),
+    ) -> Annotated[
+        list[SearchEntry],
+        Doc('Enriched events matching the search criteria.'),
+    ]:
+        """Search enriched Collective Insights events.
+
+        Endpoint:
+            `collective-insights/search`
+
+        Raises:
+            CollectiveInsightsSearchError: If connection error occurs.
+            ValidationError: If any supplied parameter is of incorrect type.
+        """
+        data = {
+            'filters': {
+                'organizations': organizations,
+                'indicator_type': indicator_type,
+                'detection_rule': detection_rule_id,
+                'detection_type': detection_type,
+                'submission_method': submission_method,
+                'detection_time': {'from': detection_time_from, 'to': detection_time_to},
+                'associated_threats': {
+                    'malware': malware_id,
+                    'mitre_code': mitre_code_id,
+                    'threat_actor': threat_actor_id,
+                },
+                'autonomous_threat_operations': {
+                    'use_case': atop_use_case,
+                    'profile': atop_profile_id,
+                    'job': atop_job_id,
+                },
+                'integration_type': integration_type_id,
+                'indicator': {'risk': {'score': {'at_detection': indicator_risk_score}}},
+            },
+            'limit': min(page_size, max_results),
+        }
+        search_data = SearchIn.model_validate(data)
+        self.log.info(f'Searching Collective Insights events with query: {search_data.json()}')
+
+        results = self.rf_client.request_paged(
+            method='post',
+            url=EP_COLLECTIVE_INSIGHTS_SEARCH,
+            data=search_data.json(),
+            results_path='data',
+            offset_key='offset',
+            max_results=max_results,
+        )
+
+        return [SearchEntry.model_validate(r) for r in results]
+
     def _prepare_ci_request(
         self,
         insight: list[Insight],

{psengine-2.12.0 → psengine-2.13.0}/psengine/collective_insights/constants.py

@@ -42,3 +42,14 @@ DETECTION_SUB_FORMAT_MAPPING = {
     'detection_type': ['detection', 'type'],
 }
 SUMMARY_DEFAULT = True
+
+SEARCH_MAX_LIMIT = 1000
+SEARCH_PAGE_SIZE = 100
+
+SUBMISSION_METHOD_API = 'api'
+SUBMISSION_METHOD_INTEGRATION = 'integration'
+SUBMISSION_METHOD_SANDBOX = 'sandbox'
+
+ATOP_USE_CASE_HUNTING = 'hunting'
+ATOP_USE_CASE_DETECTION = 'detection'
+ATOP_USE_CASE_PREVENTION = 'prevention'

{psengine-2.12.0 → psengine-2.13.0}/psengine/collective_insights/errors.py

@@ -16,3 +16,7 @@ from ..errors import RecordedFutureError
 
 class CollectiveInsightsError(RecordedFutureError):
     """Error raised when there was an error with the Collective Insights API."""
+
+
+class CollectiveInsightsSearchError(RecordedFutureError):
+    """Error raised when there was an error searching the Collective Insights API."""

{psengine-2.12.0 → psengine-2.13.0}/psengine/collective_insights/insight.py

@@ -15,16 +15,19 @@ from datetime import datetime
 from functools import total_ordering
 from typing import Annotated
 
-from pydantic import BeforeValidator
+from pydantic import BeforeValidator, model_validator
 
 from ..common_models import IdNameType, RFBaseModel
 from ..constants import TIMESTAMP_STR
 from ..helpers import Validators
+from .constants import SEARCH_PAGE_SIZE
 from .models import (
     RequestDetection,
     RequestIOC,
     RequestOptions,
+    SearchFilters,
     SubmissionResult,
+    _prune_none,
 )
 
 
@@ -94,3 +97,21 @@ class InsightsIn(RFBaseModel):
     """Validate data received from CI."""
 
     result: SubmissionResult
+
+
+class SearchIn(RFBaseModel):
+    """Validate data sent to the `/search` endpoint."""
+
+    filters: SearchFilters | None = None
+    limit: int = SEARCH_PAGE_SIZE
+
+    @model_validator(mode='before')
+    @classmethod
+    def _drop_empty_filters(cls, data):
+        """Drop `filters` entirely when caller supplied only `None` values."""
+        if not isinstance(data, dict):
+            return data
+        filters = data.get('filters')
+        if isinstance(filters, dict) and not _prune_none(filters):
+            data = {k: v for k, v in data.items() if k != 'filters'}
+        return data

psengine-2.13.0/psengine/collective_insights/models.py

@@ -0,0 +1,352 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+from datetime import datetime
+from enum import Enum
+from functools import total_ordering
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, BeforeValidator, Field, model_validator
+
+from ..common_models import DetectionRuleType, IOCType, RFBaseModel
+from ..constants import TIMESTAMP_STR
+from ..helpers.helpers import Validators
+
+
+def _to_list_or_presence(value):
+    """Coerce a scalar to a list; pass `present`/`absent` sentinels and `None` through."""
+    if value is None or value in ('present', 'absent'):
+        return value
+    return Validators.convert_str_to_list(value)
+
+
+def _prune_none(data):
+    """Recursively drop keys with `None` or empty-container values from a dict."""
+    if not isinstance(data, dict):
+        return data
+    pruned = {}
+    for key, value in data.items():
+        cleaned = _prune_none(value)
+        if cleaned is None:
+            continue
+        if isinstance(cleaned, (dict, list)) and not cleaned:
+            continue
+        pruned[key] = cleaned
+    return pruned
+
+
+class DetectionType(Enum):
+    detection_rule = 'detection_rule'
+    correlation = 'correlation'
+    playbook = 'playbook'
+    sandbox = 'sandbox'
+
+
+class SummaryProcessed(RFBaseModel):
+    ip: int
+    domain: int
+    hash_: int = Field(alias='hash')
+    vulnerability: int
+    url: int
+
+
+class ResponseSummary(RFBaseModel):
+    processed: SummaryProcessed
+
+
+class RequestOptions(RFBaseModel):
+    debug: bool = False
+    summary: bool = True
+
+
+class RequestIOC(RFBaseModel):
+    type_: IOCType = Field(alias='type')
+    value: str
+    source_type: str | None = None
+    field: str | None = None
+
+
+class RequestDetection(RFBaseModel):
+    id_: str | None = Field(alias='id', default=None)
+    name: str | None = None
+    type_: DetectionType = Field(alias='type')
+    sub_type: DetectionRuleType | None = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def validate_detection_rule(cls, data):
+        """Validate detection rule scenario.
+
+        - id must be present.
+        - sub_type must be present and should be one of DetectionRuleType
+        """
+        try:
+            detection_type = data['type']
+        except KeyError as e:
+            raise ValueError('type field is mandatory') from e
+
+        if detection_type == 'detection_rule' and not (data.get('id') and data.get('sub_type')):
+            raise ValueError(f'With {detection_type} the id and sub_type fields are mandatory')
+
+        return data
+
+
+class SubmissionResult(RFBaseModel):
+    status: str
+    debug: bool
+    summary: ResponseSummary | None = None
+
+
+class SearchDetectionType(Enum):
+    correlation = 'correlation'
+    playbook = 'playbook'
+    detection_rule = 'detection_rule'
+    sandbox = 'sandbox'
+    threat_hunt = 'threat_hunt'
+    vulnerability_scan = 'vulnerability_scan'
+
+
+class SubmissionMethod(Enum):
+    api = 'api'
+    integration = 'integration'
+    sandbox = 'sandbox'
+
+
+class ATOPUseCase(Enum):
+    hunting = 'hunting'
+    detection = 'detection'
+    prevention = 'prevention'
+
+
+Presence = Literal['present', 'absent']
+
+
+class IdFilter(RFBaseModel):
+    """Filter by a list of IDs or a `present`/`absent` sentinel.
+
+    Also accepts a bare value (str, list, `present`, `absent`) which is auto-wrapped
+    into `{'id': value}`.
+    """
+
+    id_: Annotated[
+        list[str] | Presence | None,
+        BeforeValidator(_to_list_or_presence),
+    ] = Field(alias='id', default=None)
+
+    @model_validator(mode='before')
+    @classmethod
+    def _wrap_bare_value(cls, data):
+        if data is None or isinstance(data, dict):
+            return data
+        return {'id': data}
+
+
+class DetectionTimeFilter(RFBaseModel):
+    from_: datetime | None = Field(alias='from', default=None)
+    to: datetime | None = None
+
+
+class AssociatedThreatsFilter(RFBaseModel):
+    malware: IdFilter | None = None
+    mitre_code: IdFilter | None = None
+    threat_actor: IdFilter | None = None
+
+
+class ATOPFilter(RFBaseModel):
+    use_case: Annotated[
+        list[ATOPUseCase] | Presence | None,
+        BeforeValidator(_to_list_or_presence),
+    ] = None
+    profile: IdFilter | None = None
+    job: IdFilter | None = None
+
+
+class RiskScoreRange(RFBaseModel):
+    gte: int | None = None
+    gt: int | None = None
+    lte: int | None = None
+    lt: int | None = None
+
+
+class IndicatorRiskScoreFilter(RFBaseModel):
+    at_detection: RiskScoreRange | Presence | None = None
+
+
+class IndicatorRiskFilter(RFBaseModel):
+    score: IndicatorRiskScoreFilter | None = None
+
+
+class IndicatorFilter(RFBaseModel):
+    risk: IndicatorRiskFilter | None = None
+
+
+class SearchFilters(RFBaseModel):
+    organizations: Annotated[
+        list[str] | None,
+        BeforeValidator(Validators.convert_str_to_list),
+        AfterValidator(Validators.check_uhash_prefix),
+    ] = None
+    indicator_type: Annotated[
+        list[IOCType] | Presence | None,
+        BeforeValidator(_to_list_or_presence),
+    ] = None
+    detection_rule: IdFilter | None = None
+    detection_type: Annotated[
+        list[SearchDetectionType] | Presence | None,
+        BeforeValidator(_to_list_or_presence),
+    ] = None
+    submission_method: Annotated[
+        list[SubmissionMethod] | Presence | None,
+        BeforeValidator(_to_list_or_presence),
+    ] = None
+    detection_time: DetectionTimeFilter | None = None
+    associated_threats: AssociatedThreatsFilter | None = None
+    autonomous_threat_operations: ATOPFilter | None = None
+    integration_type: IdFilter | None = None
+    indicator: IndicatorFilter | None = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def _prune_input(cls, data):
+        """Recursively drop `None` values and empty containers from filter input."""
+        return _prune_none(data) if isinstance(data, dict) else data
+
+
+class Entity(RFBaseModel):
+    id_: str = Field(alias='id')
+    type_: str | None = Field(alias='type', default=None)
+    name: str | None = None
+    display_name: str | None = Field(alias='displayName', default=None)
+
+
+class IntegrationType(RFBaseModel):
+    id_: str = Field(alias='id')
+    name: str | None = None
+    display_name: str | None = Field(alias='displayName', default=None)
+
+
+class ATOPProfile(RFBaseModel):
+    id_: str | None = Field(alias='id', default=None)
+
+
+class ATOPJob(RFBaseModel):
+    id_: str | None = Field(alias='id', default=None)
+
+
+class ATOPResult(RFBaseModel):
+    use_case: str | None = None
+    profile: ATOPProfile | None = None
+    job: ATOPJob | None = None
+
+
+class IndicatorRiskScore(RFBaseModel):
+    at_detection: int | None = None
+
+
+class IndicatorRisk(RFBaseModel):
+    score: IndicatorRiskScore | None = None
+
+
+class SearchIndicator(RFBaseModel):
+    type_: IOCType = Field(alias='type')
+    value: str
+    field: str | None = None
+    risk: IndicatorRisk | None = None
+
+
+class SearchDetectionRule(RFBaseModel):
+    id_: str = Field(alias='id')
+
+
+class SearchIncident(RFBaseModel):
+    id_: str | None = Field(alias='id', default=None)
+    name: str | None = None
+    type_: str | None = Field(alias='type', default=None)
+
+
+class SearchAssociatedThreats(RFBaseModel):
+    threat_actors: list[Entity] | None = None
+    malware: list[Entity] | None = None
+    mitre_codes: list[Entity] | None = None
+
+
+@total_ordering
+class SearchEntry(RFBaseModel):
+    """A single enriched Collective Insights event returned by `/search`.
+
+    This class supports hashing, equality comparison, string representation, and total ordering
+    of `SearchEntry` instances.
+
+    Hashing:
+        Returns a hash value based on the event `id_` and `detection_time`.
+
+    Equality:
+        Checks equality between two `SearchEntry` instances based on `id_` and `detection_time`.
+
+    Greater-than Comparison:
+        Defines a greater-than comparison between two `SearchEntry` instances based on the
+        `detection_time` and the `id_`.
+
+    String Representation:
+        Returns a string representation of the `SearchEntry` instance including the `id_`,
+        the IOC value, the detection time, and the detection type.
+
+        ```python
+        >>> print(entry)
+        Event ID: 1609a, IOC: 1.2.3.4, Detection Time: 2025-12-15 13:05:23, Detection Type: sandbox
+        ```
+
+    Ordering:
+        The ordering of `SearchEntry` instances is determined primarily by `detection_time`.
+        If two instances have the same `detection_time`, the event `id_` is used as a secondary
+        criterion.
+    """
+
+    id_: str = Field(alias='id')
+    organizations: list[str] | None = None
+    integration_type: IntegrationType | None = None
+    submission_method: str | None = None
+    detection_type: str | None = None
+    autonomous_threat_operations: ATOPResult | None = None
+    detection_time: datetime | None = None
+    indicator: SearchIndicator | None = None
+    action: str | None = None
+    action_category: str | None = None
+    detection_rules: list[SearchDetectionRule] | None = None
+    incident: SearchIncident | None = None
+    associated_threats: SearchAssociatedThreats | None = None
+    context: dict | None = None
+
+    def __hash__(self):
+        return hash((self.id_, self.detection_time))
+
+    def __eq__(self, other: 'SearchEntry'):
+        return (self.id_, self.detection_time) == (other.id_, other.detection_time)
+
+    def __gt__(self, other: 'SearchEntry'):
+        return (self.detection_time, self.id_) > (other.detection_time, other.id_)
+
+    def __str__(self):
+        ioc = self.indicator.value if self.indicator else 'N/A'
+        detection_time = (
+            self.detection_time.strftime(TIMESTAMP_STR) if self.detection_time else 'N/A'
+        )
+        return (
+            f'Event ID: {self.id_}, IOC: {ioc}, Detection Time: {detection_time}, '
+            f'Detection Type: {self.detection_type or "N/A"}'
+        )
+
+
+class SearchCounts(RFBaseModel):
+    returned: int | None = None
+    total: int | None = None

{psengine-2.12.0 → psengine-2.13.0}/psengine/common_models.py

@@ -94,6 +94,7 @@ class DetectionRuleType(Enum):
     yara = 'yara'
     snort = 'snort'
     suricata = 'suricata'
+    nuclei = 'nuclei'
 
 
 class ClearTextPassword(Secret[str]):

{psengine-2.12.0 → psengine-2.13.0}/psengine/endpoints.py

@@ -93,6 +93,7 @@ EP_DETECTION_RULES = BASE_URL + '/detection-rule/search'
 ###############################################################################
 EP_COLLECTIVE_INSIGHTS = BASE_URL + '/collective-insights'
 EP_COLLECTIVE_INSIGHTS_DETECTIONS = EP_COLLECTIVE_INSIGHTS + '/detections'
+EP_COLLECTIVE_INSIGHTS_SEARCH = EP_COLLECTIVE_INSIGHTS + '/search'
 
 ###############################################################################
 # Analyst Notes API Endpoints

{psengine-2.12.0 → psengine-2.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "psengine"
-version = "2.12.0"
+version = "2.13.0"
 readme = "README.md"
 license = "MIT"
 requires-python = ">=3.10, <3.15"

{psengine-2.12.0 → psengine-2.13.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "psengine"
-version = "2.12.0"
+version = "2.13.0"
 readme = "README.md"
 license = "MIT"
 requires-python = ">=3.10, <3.15"