pymisp 2.5.8__tar.gz → 2.5.9__tar.gz

{pymisp-2.5.8 → pymisp-2.5.9}/CHANGELOG.txt

@@ -2,11 +2,55 @@ Changelog
 =========
 
 
+v2.5.9 (2025-03-26)
+-------------------
+
+Changes
+~~~~~~~
+- Make mypy happy. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- [emailobject] Improve parsing of the Received headers. [Sami Tainio]
+
+  - Rewrite of the section that deals with the logic handling the Received headers
+  - Previous version was not picking picking up e.g. Reverse DNS hostnames at all and some of the IPs were not ever handled
+- [emailobject] Improve parsing of the Received headers. [Sami Tainio]
+
+  - Rewrite of the section that deals with the logic handling the Received headers
+  - Previous version was not picking picking up e.g. Reverse DNS hostnames at all and some of the IPs were not ever handled
+
+Other
+~~~~~
+- [chg] Satisfy tests. [Sami Tainio]
+- Improve the logging message. [Peter Leitmann]
+- Add a check if add_attributes() method truly has values to add. [Peter
+  Leitmann]
+- [chg] Satisfy tests. [Sami Tainio]
+
+
+v2.5.8.1 (2025-03-18)
+---------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump objects, version. [Raphaël Vinot]
+
+Fix
+~~~
+- Typo in objects. [Raphaël Vinot]
+
+
 v2.5.8 (2025-03-18)
 -------------------
 
 Changes
 ~~~~~~~
+- Bump changelog. [Raphaël Vinot]
 - Bump deps, version. [Raphaël Vinot]
 - Bump deps. [Raphaël Vinot]
 

{pymisp-2.5.8 → pymisp-2.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pymisp
-Version: 2.5.8
+Version: 2.5.9
 Summary: Python API for MISP.
 License: BSD-2-Clause
 Author: Raphaël Vinot
@@ -32,7 +32,7 @@ Requires-Dist: RTFDE (>=0.1.2) ; (python_version <= "3.9") and (extra == "email"
 Requires-Dist: beautifulsoup4 (>=4.13.3) ; extra == "openioc"
 Requires-Dist: deprecated (>=1.2.18)
 Requires-Dist: docutils (>=0.21.2) ; (python_version >= "3.11") and (extra == "docs")
-Requires-Dist: extract_msg (>=0.53.2) ; extra == "email"
+Requires-Dist: extract_msg (>=0.54.0) ; extra == "email"
 Requires-Dist: lief (>=0.16.4) ; extra == "fileobjects"
 Requires-Dist: myst-parser (>=4.0.1) ; (python_version >= "3.11") and (extra == "docs")
 Requires-Dist: oletools (>=0.60.2) ; extra == "email"

pymisp-2.5.9/pymisp/data/misp-objects/objects/github-repo/definition.json

@@ -0,0 +1,142 @@
+{
+  "attributes": {
+    "archived": {
+      "description": "Is the repository archived?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "created-at": {
+      "description": "Date of the repository creation",
+      "misp-attribute": "datetime",
+      "ui-priority": 0
+    },
+    "description": {
+      "description": "Repository description",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "disabled": {
+      "description": "Is the repository disabled?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "fork": {
+      "description": "Is the repository a forked repository?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "forks-count": {
+      "description": "Number of forks",
+      "misp-attribute": "counter",
+      "ui-priority": 1
+    },
+    "full-name": {
+      "description": "Full name of the repository. [Username/Repository name]",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "has-downloads": {
+      "description": "Have the repository been downloaded?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "has-wiki": {
+      "description": "Does the repository have a wiki?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "id": {
+      "description": "Repository id",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "languages": {
+      "description": "Languages used in the repository",
+      "misp-attribute": "text",
+      "multiple": true,
+      "ui-priority": 1
+    },
+    "link": {
+      "description": "Link to the GitHub repository.",
+      "misp-attribute": "link",
+      "multiple": true,
+      "ui-priority": 1
+    },
+    "name": {
+      "description": "name of the repository. [Repository name]",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "open-issues": {
+      "description": "Number of open issues",
+      "misp-attribute": "counter",
+      "ui-priority": 1
+    },
+    "private": {
+      "description": "Is the repository private?",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "True",
+        "False"
+      ],
+      "ui-priority": 1
+    },
+    "pushed-at": {
+      "description": "Date of last push",
+      "misp-attribute": "datetime",
+      "ui-priority": 0
+    },
+    "topics": {
+      "description": "Topics linked to the repository",
+      "misp-attribute": "text",
+      "multiple": true,
+      "ui-priority": 1
+    },
+    "updated-at": {
+      "description": "Date of the last update",
+      "misp-attribute": "datetime",
+      "ui-priority": 0
+    },
+    "username": {
+      "description": "Owner of the repository. [Username]",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    }
+  },
+  "description": "GitHub repository",
+  "meta-category": "misc",
+  "name": "github-repo",
+  "requiredOneOf": [
+    "name",
+    "full-name",
+    "link"
+  ],
+  "uuid": "d2e93321-3d0c-4215-88a7-62ccb56fef89",
+  "version": 2
+}

{pymisp-2.5.8 → pymisp-2.5.9}/pymisp/mispevent.py

@@ -1120,6 +1120,10 @@ class MISPObject(AnalystDataBehaviorMixin):
         Helper for object_relation when multiple is True in the template.
         It is the same as calling multiple times add_attribute with the same object_relation.
         '''
+        if not attributes:
+            logger.warning(f"No attributes provided for object relation '{object_relation}'; skipping attribute addition.")
+            return []
+
         to_return = []
         for attribute in attributes:
             if isinstance(attribute, MISPAttribute):

{pymisp-2.5.8 → pymisp-2.5.9}/pymisp/tools/emailobject.py

@@ -373,37 +373,95 @@ class EMailObject(AbstractMISPObjectGenerator):
                 # email object doesn't support display name for all email addrs
                 pass
 
+    def extract_matches(self, pattern: re.Pattern[str], text: str) -> list[tuple[str, ...]]:
+        """Returns all regex matches for a given pattern in a text."""
+        return re.findall(pattern, text)
+
+    def add_ip_attribute(self, ip_candidate: str, received: str, seen_attributes: set[tuple[str, str]]) -> None:
+        """Validates and adds an IP address to MISP if it's public and not already seen during extraction."""
+        try:
+            ip = ipaddress.ip_address(ip_candidate)
+            if not ip.is_private and ("received-header-ip", ip_candidate) not in seen_attributes:
+                self.add_attribute("received-header-ip", ip_candidate, comment=received)
+                seen_attributes.add(("received-header-ip", ip_candidate))
+        except ValueError:
+            pass  # Invalid IPs are ignored
+
+    def add_hostname_attribute(self, hostname: str, received: str, seen_attributes: set[tuple[str, str]]) -> None:
+        """Validates and adds a hostname to MISP if it contains a valid TLD-like format and is not already seen."""
+        if "." in hostname and not hostname.endswith(".") and len(hostname.split(".")[-1]) > 1:
+            if ("received-header-hostname", hostname) not in seen_attributes:
+                self.add_attribute("received-header-hostname", hostname, comment=received)
+                seen_attributes.add(("received-header-hostname", hostname))
+
+    def process_received_header(self, received: str, seen_attributes: set[tuple[str, str]]) -> None:
+        """Processes a single 'Received' header and extracts hostnames and IPs."""
+
+        # Regex patterns
+        received_from_regex = re.compile(
+            r'from\s+([\w.-]+)'  # Declared sending hostname
+            r'(?:\s+\(([^)]+)\))?'  # Reverse DNS hostname inside parentheses
+        )
+        ipv4_regex = re.compile(
+            r'\[(?P<ipv4_brackets>(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9]))\]'  # IPv4 inside []
+            r'|\((?P<ipv4_parentheses>(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9]))\)'  # IPv4 inside ()
+            r'|(?<=\.\s)(?P<ipv4_after_domain>(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.'
+            r'(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9]))\b'  # IPv4 appearing after a domain.
+        )
+        ipv6_regex = re.compile(
+            r'\b(?:[a-fA-F0-9]{1,4}:[a-fA-F0-9]{1,4}(?::[a-fA-F0-9]{1,4}){0,6})\b'
+        )
+
+        # Extract hostnames
+        matches = self.extract_matches(received_from_regex, received)
+        for match in matches:
+            declared_sending_host = match[0].strip() if match[0] else None
+            reverse_dns_host = match[1].split()[0].strip("[]()").rstrip('.') if match[1] else None
+
+            if declared_sending_host:
+                clean_host = declared_sending_host.strip("[]()")
+                try:
+                    ipaddress.ip_address(declared_sending_host)
+                    self.add_ip_attribute(declared_sending_host, received, seen_attributes)
+                except ValueError:
+                    self.add_hostname_attribute(declared_sending_host, received, seen_attributes)
+
+            if reverse_dns_host:
+                try:
+                    ipaddress.ip_address(reverse_dns_host)
+                    self.add_ip_attribute(reverse_dns_host, received, seen_attributes)
+                except ValueError:
+                    self.add_hostname_attribute(reverse_dns_host, received, seen_attributes)
+
+        # Extract and add **only valid** IPv4 addresses
+        for ipv4_match in self.extract_matches(ipv4_regex, received):
+            ip_candidate = ipv4_match[0] or ipv4_match[1] or ipv4_match[2]  # Select first non-empty match
+            if ip_candidate:
+                self.add_ip_attribute(ip_candidate, received, seen_attributes)
+
+        # Extract and add IPv6 addresses
+        for ipv6_match in self.extract_matches(ipv6_regex, received):
+            self.add_ip_attribute(ipv6_match, received, seen_attributes)
+
     def __generate_received(self) -> None:
         """
-        Extract IP addresses from received headers that are not private. Also extract hostnames or domains.
+        Extracts public IP addresses and hostnames from "Received" email headers.
         """
-        received_items = self.email.get_all("received")
-        if received_items is None:
-            return
-        for received in received_items:
-            fromstr = re.split(r"\sby\s", received)[0].strip()
-            if fromstr.startswith('from') is not True:
-                continue
-            for i in ['(', ')', '[', ']']:
-                fromstr = fromstr.replace(i, " ")
-            tokens = fromstr.split(" ")
-            ip = None
-            for token in tokens:
-                try:
-                    ip = ipaddress.ip_address(token)
-                    break
-                except ValueError:
-                    pass  # token is not IP address
 
-            if not ip or ip.is_private:
-                continue  # skip header if IP not found or is private
+        received_items = self.email.get_all("Received")
+        if not received_items:
+            return
 
-            self.add_attribute("received-header-ip", value=str(ip), comment=fromstr)
+        # Track added attributes to prevent duplicates (store as (type, value) tuples)
+        seen_attributes: set[tuple[str, str]] = set()
 
-        # The hostnames and/or domains always come after the "Received: from"
-        # part so we can use regex to pick up those attributes.
-        received_from = re.findall(r'(?<=from\s)[\w\d\.\-]+\.\w{2,24}', str(received_items))
-        try:
-            [self.add_attribute("received-header-hostname", i) for i in received_from]
-        except Exception:
-            pass
+        for received in received_items:
+            self.process_received_header(received, seen_attributes)

{pymisp-2.5.8 → pymisp-2.5.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pymisp"
-version = "2.5.8"
+version = "2.5.9"
 description = "Python API for MISP."
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -46,7 +46,7 @@ pdfexport = [ "reportlab (>=4.3.1)" ]
 url = [ "pyfaup (>=1.2)" ]
 
 email = [
-    "extract_msg (>=0.53.2)",
+    "extract_msg (>=0.54.0)",
     "RTFDE (>=0.1.2) ; python_version <= \"3.9\"",
     "oletools (>=0.60.2)"
 ]