psengine 2.11.0__tar.gz → 2.12.0__tar.gz

{psengine-2.11.0 → psengine-2.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: psengine
-Version: 2.11.0
+Version: 2.12.0
 Summary: psengine is a simple, yet elegant, library for rapid development of integrations with Recorded Future.
 Keywords: API,Recorded Future,Cyber Security Engineering,Threat Intelligence
 Author: Moise Medici, Patrick Kinsella, Ernest Bartosevic
@@ -93,3 +93,7 @@ PSEngine has been made public from our internal version 2.0.4. Previous versions
 
 The documentation arrived at version 2.1.1. Older versions are not explicitly documented and changes can be found in the [Release History](./CHANGELOG.md) section.
 
+## Contributing
+
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup, code standards, testing conventions, and the CI workflow.
+

{psengine-2.11.0 → psengine-2.12.0}/README.md

@@ -60,3 +60,7 @@ PSEngine has been made public from our internal version 2.0.4. Previous versions
 
 The documentation arrived at version 2.1.1. Older versions are not explicitly documented and changes can be found in the [Release History](./CHANGELOG.md) section.
 
+## Contributing
+
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup, code standards, testing conventions, and the CI workflow.
+

{psengine-2.11.0 → psengine-2.12.0}/psengine/analyst_notes/note_mgr.py

@@ -28,7 +28,7 @@ from ..endpoints import (
     EP_ANALYST_NOTE_PUBLISH,
     EP_ANALYST_NOTE_SEARCH,
 )
-from ..helpers import debug_call
+from ..helpers import debug_call, validate_list
 from ..helpers.helpers import connection_exceptions
 from ..rf_client import RFClient
 from .constants import NOTES_PER_PAGE
@@ -358,4 +358,4 @@ class AnalystNoteMgr:
             max_results=max_results,
         )
 
-        return [AnalystNote.model_validate(d) for d in response]
+        return validate_list(AnalystNote, response, id_path='id', log=self.log)

{psengine-2.11.0 → psengine-2.12.0}/psengine/enrich/soar_mgr.py

@@ -19,7 +19,7 @@ from pydantic import validate_call
 from typing_extensions import Doc
 
 from ..endpoints import EP_SOAR_ENRICHMENT
-from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call
+from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call, validate_list
 from ..rf_client import RFClient
 from .constants import SOAR_POST_ROWS
 from .errors import EnrichmentSoarError
@@ -158,11 +158,5 @@ class SoarMgr:
         res = self.rf_client.request('post', EP_SOAR_ENRICHMENT, data=data.json()).json()['data'][
             'results'
         ]
-        result = []
-
-        for d in res:
-            content = SOAREnrichedEntity.model_validate(d)
-            entity = content.entity.name
-            result.append(SOAREnrichOut(entity=entity, is_enriched=True, content=content))
-
-        return result
+        contents = validate_list(SOAREnrichedEntity, res, id_path='entity.name', log=self.log)
+        return [SOAREnrichOut(entity=c.entity.name, is_enriched=True, content=c) for c in contents]

{psengine-2.11.0 → psengine-2.12.0}/psengine/entity_lists/entity_list.py

@@ -24,7 +24,7 @@ from ..common_models import IdNameType, RFBaseModel
 from ..constants import TIMESTAMP_STR
 from ..endpoints import EP_LIST, EP_LIST_ENTITIES_WITH_TAGS, EP_LIST_ENTITY_TAGS
 from ..entity_match import EntityMatchMgr, MatchApiError
-from ..helpers import debug_call
+from ..helpers import debug_call, validate_list
 from ..helpers.helpers import connection_exceptions
 from ..rf_client import RFClient
 from .constants import ADD_OP, ERROR_NAME, IS_READY_INCREMENT, REMOVE_OP, UNCHANGED_NAME
@@ -286,7 +286,7 @@ class EntityList(ListInfoOut):
         """
         url = EP_LIST + '/' + self.id_ + '/entities'
         response = self.rf_client.request('get', url)
-        return [ListEntity.model_validate(entity) for entity in response.json()]
+        return validate_list(ListEntity, response.json(), id_path='entity.name', log=self.log)
 
     @debug_call
     @connection_exceptions(ignore_status_code=[], exception_to_raise=ListApiError)
@@ -319,7 +319,9 @@ class EntityList(ListInfoOut):
         """
         url = EP_LIST_ENTITIES_WITH_TAGS.format(self.id_)
         response = self.rf_client.request('get', url)
-        return [ListEntityWithTags.model_validate(entity) for entity in response.json()]
+        return validate_list(
+            ListEntityWithTags, response.json(), id_path='entity.name', log=self.log
+        )
 
     @debug_call
     @validate_call

{psengine-2.11.0 → psengine-2.12.0}/psengine/entity_match/entity_match_mgr.py

@@ -21,7 +21,7 @@ from typing_extensions import Doc
 from ..common_models import IdNameType
 from ..constants import DEFAULT_LIMIT, DEFAULT_MAX_WORKERS
 from ..endpoints import EP_ENTITY_LOOKUP, EP_ENTITY_MATCH
-from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call
+from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call, validate_list
 from ..rf_client import RFClient
 from .entity_match import EntityLookup, EntityMatchIn, ResolvedEntity
 from .errors import MatchApiError
@@ -63,7 +63,7 @@ class EntityMatchMgr:
 
         request_body = EntityMatchIn(name=entity_name, type=entity_type, limit=limit)
         response = self.rf_client.request('post', EP_ENTITY_MATCH, data=request_body.json())
-        response = [IdNameType.model_validate(d) for d in response.json()]
+        response = validate_list(IdNameType, response.json(), id_path='name', log=self.log)
         return (
             list({ResolvedEntity(entity=d.name, is_found=bool(d.id_), content=d) for d in response})
             if response

{psengine-2.11.0 → psengine-2.12.0}/psengine/helpers/__init__.py

@@ -22,3 +22,4 @@ from .helpers import (
     debug_call,
     dump_models,
 )
+from .validation import validate_list

psengine-2.12.0/psengine/helpers/validation.py

@@ -0,0 +1,111 @@
+##################################### TERMS OF USE ###########################################
+# The following code is provided for demonstration purpose only, and should not be used      #
+# without independent verification. Recorded Future makes no representations or warranties,  #
+# express, implied, statutory, or otherwise, regarding any aspect of this code or of the     #
+# information it may retrieve, and provides it both strictly “as-is” and without assuming    #
+# responsibility for any information it may retrieve. Recorded Future shall not be liable    #
+# for, and you assume all risk of using, the foregoing. By using this code, Customer         #
+# represents that it is solely responsible for having all necessary licenses, permissions,   #
+# rights, and/or consents to connect to third party APIs, and that it is solely responsible  #
+# for having all necessary licenses, permissions, rights, and/or consents to any data        #
+# accessed from any third party API.                                                         #
+##############################################################################################
+
+import logging
+from collections.abc import Iterable
+from typing import Annotated, Any, TypeVar
+
+from pydantic import BaseModel, TypeAdapter, ValidationError
+from typing_extensions import Doc
+
+LOG = logging.getLogger(__name__)
+
+T = TypeVar('T', bound=BaseModel)
+
+
+def validate_list(
+    model_cls: Annotated[type[T], Doc('Pydantic model each item should validate against.')],
+    items: Annotated[
+        Iterable[Any], Doc('Raw items to validate, typically dicts from an API response.')
+    ],
+    id_path: Annotated[
+        str | None,
+        Doc(
+            'Dotted path into each raw item used to build a friendly identifier for log lines '
+            "and exception notes, e.g. `'entity.name'`. If unresolvable for a given item, only "
+            'the index is reported.'
+        ),
+    ] = None,
+    log: Annotated[
+        logging.Logger | None,
+        Doc("Logger for the warnings. Defaults to this module's logger."),
+    ] = None,
+) -> Annotated[list[T], Doc('The list of validated `model_cls` instances.')]:
+    """Validate a list of dicts against `model_cls`.
+
+    Uses pydantic's `TypeAdapter` so a single validation pass collects all per-item errors with an
+    index in their `loc`.
+
+    Before re-raising, each failing item is (a) logged individually as a warning and (b) on
+    Python 3.11+ attached to the exception via `add_note`, so the entity identifier shows up in
+    the traceback too. On Python 3.10 the note step is skipped since `BaseException.add_note`
+    is not available.
+
+    If `id_path` is provided, a human-readable identifier is extracted from the raw dict
+    (e.g. `entity.name`); otherwise only the index is reported.
+
+    Raises:
+        pydantic.ValidationError: If any item fails validation. Unchanged
+            from `[model_cls.model_validate(x) for x in items]`.
+    """
+    if not isinstance(items, list):
+        items = list(items)
+    try:
+        return TypeAdapter(list[model_cls]).validate_python(items)
+    except ValidationError as e:
+        for msg in _format_failures(e, items, id_path, model_cls):
+            (log or LOG).warning(msg)
+            if hasattr(e, 'add_note'):
+                e.add_note(msg)
+        raise
+
+
+def _format_failures(
+    exc: ValidationError,
+    items: list[Any],
+    id_path: str | None,
+    model_cls: type[BaseModel],
+) -> list[str]:
+    """Build one message per failing item, keyed by index and optional id."""
+    seen: set[int] = set()
+    messages: list[str] = []
+    for err in exc.errors():
+        loc = err.get('loc') or ()
+        if not loc or not isinstance(loc[0], int):
+            continue
+        idx = loc[0]
+        if idx in seen:
+            continue
+        seen.add(idx)
+
+        raw = items[idx] if 0 <= idx < len(items) else None
+        identifier = _resolve_id_path(raw, id_path) if id_path else None
+        id_hint = f' ({id_path}={identifier})' if identifier is not None else ''
+        field_path = '.'.join(str(p) for p in loc[1:]) or '<root>'
+        messages.append(
+            f'{model_cls.__name__} validation failed at index {idx}{id_hint}: '
+            f"field '{field_path}' {err.get('msg', '')}"
+        )
+    return messages
+
+
+def _resolve_id_path(item: Any, id_path: str) -> Any:
+    """Walk `id_path` (dot-separated) into `item`. Returns None on miss."""
+    value: Any = item
+    for part in id_path.split('.'):
+        if not isinstance(value, dict):
+            return None
+        value = value.get(part)
+        if value is None:
+            return None
+    return value

{psengine-2.11.0 → psengine-2.12.0}/psengine/identity/identity_mgr.py

@@ -29,7 +29,7 @@ from ..endpoints import (
     EP_IDENTITY_IP_LOOKUP,
     EP_IDENTITY_PASSWORD_LOOKUP,
 )
-from ..helpers import TimeHelpers, connection_exceptions, debug_call
+from ..helpers import TimeHelpers, connection_exceptions, debug_call, validate_list
 from ..rf_client import RFClient
 from .constants import DETECTIONS_PER_PAGE, MAXIMUM_IDENTITIES
 from .errors import (
@@ -252,7 +252,7 @@ class IdentityMgr:
             results_path='identities',
             max_results=max_results or DEFAULT_LIMIT,
         )
-        resp = [LeakedIdentity.model_validate(identity) for identity in resp]
+        resp = validate_list(LeakedIdentity, resp, log=self.log)
         self.log.info(f'Returned {len(resp)} identities')
         return resp
 
@@ -321,7 +321,7 @@ class IdentityMgr:
         resp = self.rf_client.request('post', url=EP_IDENTITY_PASSWORD_LOOKUP, data=data).json()[
             'results'
         ]
-        resp = [PasswordLookup.model_validate(v) for v in resp]
+        resp = validate_list(PasswordLookup, resp, id_path='password.hash_prefix', log=self.log)
         self.log.info(f'Returned {len(resp)} passwords')
         return resp
 
@@ -436,7 +436,7 @@ class IdentityMgr:
             max_results=max_results or DEFAULT_LIMIT,
             results_path='identities',
         )
-        resp = [LeakedIdentity.model_validate(identity) for identity in resp]
+        resp = validate_list(LeakedIdentity, resp, log=self.log)
         self.log.info(f'Returned {len(resp)} identities')
         return resp
 
@@ -561,7 +561,7 @@ class IdentityMgr:
             max_results=max_results or DEFAULT_LIMIT,
             results_path='identities',
         )
-        resp = [LeakedIdentity.model_validate(identity) for identity in resp]
+        resp = validate_list(LeakedIdentity, resp, log=self.log)
         self.log.info(f'Returned {len(resp)} identities')
         return resp
 
@@ -663,7 +663,7 @@ class IdentityMgr:
             max_results=max_results or DEFAULT_LIMIT,
             results_path='identities',
         )
-        resp = [CredentialSearch.model_validate(d) for d in resp]
+        resp = validate_list(CredentialSearch, resp, id_path='login', log=self.log)
         self.log.info(f'Returned {len(resp)} credentials')
         return resp
 
@@ -707,7 +707,7 @@ class IdentityMgr:
         resp = self.rf_client.request('post', url=EP_IDENTITY_DUMP_SEARCH, data=payload).json()[
             'dumps'
         ]
-        resp = [DumpSearchOut.model_validate(d) for d in resp]
+        resp = validate_list(DumpSearchOut, resp, id_path='name', log=self.log)
         self.log.info(f'Returned {len(resp)} dump results')
         return resp
 

{psengine-2.11.0 → psengine-2.12.0}/psengine/risk_history/risk_history_mgr.py

@@ -18,7 +18,7 @@ from pydantic import validate_call
 from typing_extensions import Doc
 
 from ..endpoints import EP_RISK_HISTORY
-from ..helpers import debug_call
+from ..helpers import debug_call, validate_list
 from ..helpers.helpers import connection_exceptions
 from ..rf_client import RFClient
 from .errors import RiskHistoryError
@@ -57,4 +57,4 @@ class RiskHistoryMgr:
         data = RiskHistoryIn.model_validate({'entities': entities, 'from': from_, 'to': to})
         attrs = self.rf_client.request('post', EP_RISK_HISTORY, data=data.json()).json()['data']
 
-        return [RiskHistory.model_validate(attr) for attr in attrs]
+        return validate_list(RiskHistory, attrs, id_path='entity.id', log=self.log)

{psengine-2.11.0 → psengine-2.12.0}/psengine/risk_rules/risk_rule_mgr.py

@@ -18,7 +18,7 @@ from pydantic import validate_call
 from typing_extensions import Doc
 
 from ..endpoints import EP_RISK_RULES
-from ..helpers import debug_call
+from ..helpers import debug_call, validate_list
 from ..helpers.helpers import connection_exceptions
 from ..rf_client import RFClient
 from .errors import RiskRuleFetchError
@@ -60,4 +60,4 @@ class RiskRuleMgr:
         self.log.info(f'Fetching risk rules for entity type: {entity_type.value}')
         response = self.rf_client.request('get', url).json()
         results = response['data']['results']
-        return [RiskRule.model_validate(r) for r in results]
+        return validate_list(RiskRule, results, id_path='name', log=self.log)

{psengine-2.11.0 → psengine-2.12.0}/psengine/threat_maps/threat_map_mgr.py

@@ -25,7 +25,7 @@ from ..endpoints import (
     EP_THREAT_MAP_ORG,
     EP_THREAT_MAPS_LIST,
 )
-from ..helpers import debug_call
+from ..helpers import debug_call, validate_list
 from ..helpers.helpers import connection_exceptions
 from ..rf_client import RFClient
 from .errors import (
@@ -73,7 +73,7 @@ class ThreatMapMgr:
             ThreatMapInfoError: If connection error occurs.
         """
         maps_response = self.rf_client.request(method='get', url=EP_THREAT_MAPS_LIST).json()['data']
-        return [ThreatMapInfo.model_validate(entry) for entry in maps_response]
+        return validate_list(ThreatMapInfo, maps_response, id_path='name', log=self.log)
 
     @debug_call
     @validate_call
@@ -94,7 +94,7 @@ class ThreatMapMgr:
         map_type = ThreatMapType(map_type)
         url = EP_CATEGORIES.format(map_type.category_slug)
         cat_response = self.rf_client.request(method='get', url=url).json()['data']
-        return [EntityCategory.model_validate(ent) for ent in cat_response]
+        return validate_list(EntityCategory, cat_response, id_path='id', log=self.log)
 
     @debug_call
     @validate_call
@@ -134,7 +134,7 @@ class ThreatMapMgr:
             offset_key='offset',
             max_results=max_results or DEFAULT_LIMIT,
         )
-        return [ThreatActorProfile.model_validate(ta) for ta in search_response]
+        return validate_list(ThreatActorProfile, search_response, id_path='id', log=self.log)
 
     @debug_call
     @validate_call

{psengine-2.11.0 → psengine-2.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "psengine"
-version = "2.11.0"
+version = "2.12.0"
 readme = "README.md"
 license = "MIT"
 requires-python = ">=3.10, <3.15"

{psengine-2.11.0 → psengine-2.12.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "psengine"
-version = "2.11.0"
+version = "2.12.0"
 readme = "README.md"
 license = "MIT"
 requires-python = ">=3.10, <3.15"