splunk-soar-sdk 3.4.0__py3-none-any.whl → 3.6.0__py3-none-any.whl

soar_sdk/extras/email/rfc5322.py

@@ -0,0 +1,335 @@
+import email
+import re
+from dataclasses import dataclass, field
+from email.header import decode_header, make_header
+from email.message import Message
+from html import unescape
+from typing import Any
+from urllib.parse import urlparse
+
+from bs4 import BeautifulSoup, UnicodeDammit  # type: ignore[attr-defined]
+
+from soar_sdk.extras.email.utils import clean_url, decode_uni_string, is_ip
+from soar_sdk.logging import getLogger
+
+logger = getLogger()
+
+URI_REGEX = r"[Hh][Tt][Tt][Pp][Ss]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
+EMAIL_REGEX = r"\b[A-Z0-9._%+-]+@+[A-Z0-9.-]+\.[A-Z]{2,}\b"
+
+
+@dataclass
+class EmailHeaders:
+    """Extracted email headers from an RFC 5322 message."""
+
+    email_id: str | None = None
+    message_id: str | None = None
+    to: str | None = None
+    from_address: str | None = None
+    subject: str | None = None
+    date: str | None = None
+    received: list[str] = field(default_factory=list)
+    cc: str | None = None
+    bcc: str | None = None
+    x_mailer: str | None = None
+    x_priority: str | None = None
+    reply_to: str | None = None
+    content_type: str | None = None
+    raw_headers: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EmailBody:
+    """Extracted email body content."""
+
+    plain_text: str | None = None
+    html: str | None = None
+    charset: str | None = None
+
+
+@dataclass
+class EmailAttachment:
+    """Extracted email attachment metadata."""
+
+    filename: str
+    content_type: str | None = None
+    size: int = 0
+    content_id: str | None = None
+    content: bytes | None = None
+    is_inline: bool = False
+
+
+@dataclass
+class RFC5322EmailData:
+    """Complete extracted data from an RFC 5322 email message."""
+
+    raw_email: str
+    headers: EmailHeaders
+    body: EmailBody
+    urls: list[str] = field(default_factory=list)
+    attachments: list[EmailAttachment] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary representation."""
+        return {
+            "raw_email": self.raw_email,
+            "headers": {
+                "email_id": self.headers.email_id,
+                "message_id": self.headers.message_id,
+                "to": self.headers.to,
+                "from": self.headers.from_address,
+                "subject": self.headers.subject,
+                "date": self.headers.date,
+                "received": self.headers.received,
+                "cc": self.headers.cc,
+                "bcc": self.headers.bcc,
+                "x_mailer": self.headers.x_mailer,
+                "x_priority": self.headers.x_priority,
+                "reply_to": self.headers.reply_to,
+                "content_type": self.headers.content_type,
+                "raw_headers": self.headers.raw_headers,
+            },
+            "body": {
+                "plain_text": self.body.plain_text,
+                "html": self.body.html,
+                "charset": self.body.charset,
+            },
+            "urls": self.urls,
+            "attachments": [
+                {
+                    "filename": att.filename,
+                    "content_type": att.content_type,
+                    "size": att.size,
+                    "content_id": att.content_id,
+                    "is_inline": att.is_inline,
+                }
+                for att in self.attachments
+            ],
+        }
+
+
+def _decode_header_value(value: str | None) -> str | None:
+    if not value:
+        return None
+    try:
+        return str(make_header(decode_header(value)))
+    except Exception:
+        return decode_uni_string(value, value)
+
+
+def _get_charset(part: Message) -> str:
+    charset = part.get_content_charset()
+    return charset if charset else "utf-8"
+
+
+def _decode_payload(payload: bytes, charset: str) -> str:
+    try:
+        return UnicodeDammit(payload).unicode_markup.encode("utf-8").decode("utf-8")
+    except Exception:
+        try:
+            return payload.decode(charset)
+        except Exception:
+            return payload.decode("utf-8", errors="replace")
+
+
+def _extract_urls_from_content(content: str, urls: set[str], is_html: bool) -> None:
+    if is_html:
+        try:
+            soup = BeautifulSoup(content, "html.parser")
+            for link in soup.find_all(href=True):
+                href = link["href"]
+                if href and not href.startswith("mailto:"):
+                    cleaned = clean_url(href)
+                    if cleaned.startswith("http"):
+                        urls.add(cleaned)
+            for src in soup.find_all(src=True):
+                src_val = src["src"]
+                if src_val:
+                    cleaned = clean_url(src_val)
+                    if cleaned.startswith("http"):
+                        urls.add(cleaned)
+        except Exception as e:
+            logger.debug(f"Error parsing HTML for URLs: {e}")
+
+    content = unescape(content)
+    uri_matches = re.findall(URI_REGEX, content)
+    for uri in uri_matches:
+        urls.add(clean_url(uri))
+
+
+def extract_email_headers(mail: Message, email_id: str | None = None) -> EmailHeaders:
+    """Extract headers from a parsed email Message."""
+    headers = EmailHeaders()
+    headers.email_id = email_id
+    headers.message_id = mail.get("Message-ID")
+    headers.to = _decode_header_value(mail.get("To"))
+    headers.from_address = _decode_header_value(mail.get("From"))
+    headers.subject = _decode_header_value(mail.get("Subject"))
+    headers.date = mail.get("Date")
+    headers.cc = _decode_header_value(mail.get("CC"))
+    headers.bcc = _decode_header_value(mail.get("BCC"))
+    headers.x_mailer = mail.get("X-Mailer")
+    headers.x_priority = mail.get("X-Priority")
+    headers.reply_to = _decode_header_value(mail.get("Reply-To"))
+    headers.content_type = mail.get("Content-Type")
+
+    received_headers = mail.get_all("Received") or []
+    headers.received = [str(r) for r in received_headers]
+
+    for key, value in mail.items():
+        if key.lower() == "received":
+            continue
+        headers.raw_headers[key] = _decode_header_value(str(value)) if value else None
+
+    return headers
+
+
+def extract_email_body(mail: Message) -> EmailBody:
+    """Extract plain text and HTML body from a parsed email Message."""
+    body = EmailBody()
+    charset = _get_charset(mail)
+    body.charset = charset
+
+    if not mail.is_multipart():
+        payload = mail.get_payload(decode=True)
+        if payload and isinstance(payload, bytes):
+            content_type = mail.get_content_type()
+            decoded = _decode_payload(payload, charset)
+            if content_type == "text/html":
+                body.html = decoded
+            else:
+                body.plain_text = decoded
+        return body
+
+    for part in mail.walk():
+        if part.is_multipart():
+            continue
+
+        content_type = part.get_content_type()
+        content_disp = str(part.get("Content-Disposition") or "")
+
+        if "attachment" in content_disp.lower():
+            continue
+
+        payload = part.get_payload(decode=True)
+        if not payload or not isinstance(payload, bytes):
+            continue
+
+        part_charset = _get_charset(part)
+        decoded = _decode_payload(payload, part_charset)
+
+        if content_type == "text/plain" and not body.plain_text:
+            body.plain_text = decoded
+        elif content_type == "text/html" and not body.html:
+            body.html = decoded
+
+    return body
+
+
+def extract_email_urls(mail: Message) -> list[str]:
+    """Extract all URLs from email body content."""
+    urls: set[str] = set()
+    body = extract_email_body(mail)
+
+    if body.html:
+        _extract_urls_from_content(body.html, urls, is_html=True)
+    if body.plain_text:
+        _extract_urls_from_content(body.plain_text, urls, is_html=False)
+
+    return sorted(urls)
+
+
+def extract_email_attachments(
+    mail: Message, include_content: bool = False
+) -> list[EmailAttachment]:
+    """Extract attachment metadata from a parsed email Message."""
+    attachments: list[EmailAttachment] = []
+
+    if not mail.is_multipart():
+        return attachments
+
+    for part in mail.walk():
+        if part.is_multipart():
+            continue
+
+        content_disp = str(part.get("Content-Disposition") or "")
+        content_type = part.get_content_type()
+        content_id = part.get("Content-ID")
+
+        filename = part.get_filename()
+        if not filename:
+            if "attachment" not in content_disp.lower():
+                continue
+            filename = "unnamed_attachment"
+
+        filename = _decode_header_value(filename) or filename
+        is_inline = "inline" in content_disp.lower()
+        raw_payload = part.get_payload(decode=True)
+        payload = raw_payload if isinstance(raw_payload, bytes) else None
+
+        attachment = EmailAttachment(
+            filename=filename,
+            content_type=content_type,
+            size=len(payload) if payload else 0,
+            content_id=content_id.strip("<>") if content_id else None,
+            is_inline=is_inline,
+        )
+
+        if include_content and payload:
+            attachment.content = payload
+
+        attachments.append(attachment)
+
+    return attachments
+
+
+def extract_rfc5322_email_data(
+    rfc822_email: str,
+    email_id: str | None = None,
+    include_attachment_content: bool = False,
+) -> RFC5322EmailData:
+    """Extract all components from an RFC 5322 email string."""
+    mail = email.message_from_string(rfc822_email)
+
+    return RFC5322EmailData(
+        raw_email=rfc822_email,
+        headers=extract_email_headers(mail, email_id),
+        body=extract_email_body(mail),
+        urls=extract_email_urls(mail),
+        attachments=extract_email_attachments(mail, include_attachment_content),
+    )
+
+
+def extract_domains_from_urls(urls: list[str]) -> list[str]:
+    """Extract unique domains from a list of URLs."""
+    domains: set[str] = set()
+
+    for url in urls:
+        try:
+            parsed = urlparse(url)
+            if parsed.netloc and not is_ip(parsed.netloc):
+                domain = parsed.netloc.split(":")[0]
+                domains.add(domain)
+        except Exception as e:
+            logger.debug(f"Failed to parse URL for domain extraction: {e}")
+            continue
+
+    return sorted(domains)
+
+
+def extract_email_addresses_from_body(mail: Message) -> list[str]:
+    """Extract email addresses found in the email body."""
+    addresses: set[str] = set()
+    body = extract_email_body(mail)
+
+    content = ""
+    if body.plain_text:
+        content += body.plain_text
+    if body.html:
+        content += body.html
+
+    if content:
+        matches = re.findall(EMAIL_REGEX, content, re.IGNORECASE)
+        addresses.update(m.lower() for m in matches)
+
+    return sorted(addresses)

soar_sdk/extras/email/utils.py

@@ -0,0 +1,178 @@
+import hashlib
+import ipaddress
+import json
+import re
+from email.header import decode_header, make_header
+from pathlib import Path
+from typing import Any
+
+from bs4 import UnicodeDammit  # type: ignore[attr-defined]
+
+from soar_sdk.logging import getLogger
+
+logger = getLogger()
+
+FILE_EXTENSIONS = {
+    ".vmsn": ["os memory dump", "vm snapshot file"],
+    ".vmss": ["os memory dump", "vm suspend file"],
+    ".js": ["javascript"],
+    ".doc": ["doc"],
+    ".docx": ["doc"],
+    ".xls": ["xls"],
+    ".xlsx": ["xls"],
+}
+
+MAGIC_FORMATS = [
+    ("^PE.* Windows", ["pe file", "hash"]),
+    ("^MS-DOS executable", ["pe file", "hash"]),
+    ("^PDF ", ["pdf"]),
+    ("^MDMP crash", ["process dump"]),
+    ("^Macromedia Flash", ["flash"]),
+]
+
+
+def get_file_contains(file_path: str) -> list[str]:
+    """Get file type contains based on extension and magic bytes."""
+    try:
+        import magic  # type: ignore[import-not-found]
+    except ImportError:
+        logger.warning(
+            "python-magic not installed, file type detection will be limited"
+        )
+        return []
+
+    contains = []
+    ext = Path(file_path).suffix
+    contains.extend(FILE_EXTENSIONS.get(ext, []))
+
+    try:
+        magic_str = magic.from_file(file_path)
+        for regex_pattern, cur_contains in MAGIC_FORMATS:
+            if re.match(regex_pattern, magic_str):
+                contains.extend(cur_contains)
+    except Exception as e:
+        logger.debug(f"Failed to detect file type with magic: {e}")
+
+    return contains
+
+
+def is_ip(input_ip: str) -> bool:
+    """Check if input is a valid IP address."""
+    try:
+        ipaddress.ip_address(input_ip)
+        return True
+    except ValueError:
+        return False
+
+
+def is_ipv6(input_ip: str) -> bool:
+    """Validate if input is an IPv6 address."""
+    try:
+        ip = ipaddress.ip_address(input_ip)
+        return ip.version == 6
+    except ValueError:
+        return False
+
+
+def is_sha1(input_str: str) -> bool:
+    """Validate if the input is a SHA1 hash."""
+    sha1_regex = r"^[0-9a-fA-F]{40}$"
+    return bool(re.match(sha1_regex, input_str))
+
+
+def clean_url(url: str) -> str:
+    """Clean and normalize a URL string."""
+    url = url.strip(">),.]\r\n")
+    if "<" in url:
+        url = url[: url.find("<")]
+    if ">" in url:
+        url = url[: url.find(">")]
+    url = url.rstrip("]")
+    return url.strip()
+
+
+def decode_uni_string(input_str: str, def_name: str) -> str:
+    """Decode RFC 2047 encoded strings."""
+    encoded_strings = re.findall(r"=\?.*?\?=", input_str, re.I)
+
+    if not encoded_strings:
+        return input_str
+
+    try:
+        decoded_strings = [decode_header(x)[0] for x in encoded_strings]
+        decoded_string_dicts = [
+            {"value": x[0], "encoding": x[1]} for x in decoded_strings
+        ]
+    except Exception as e:
+        logger.debug(f"Decoding: {encoded_strings}. Error: {e}")
+        return def_name
+
+    new_str = ""
+    new_str_create_count = 0
+    for _i, decoded_string_dict in enumerate(decoded_string_dicts):
+        value = decoded_string_dict.get("value")
+        encoding = decoded_string_dict.get("encoding")
+
+        if not encoding or not value:
+            continue
+
+        try:
+            if encoding != "utf-8":
+                value = str(value, encoding)
+        except Exception as e:
+            logger.debug(f"Encoding conversion failed: {e}")
+
+        try:
+            new_str += UnicodeDammit(value).unicode_markup
+            new_str_create_count += 1
+        except Exception as e:
+            logger.debug(f"Unicode markup conversion failed: {e}")
+
+    if new_str and new_str_create_count == len(encoded_strings):
+        logger.debug(
+            "Creating a new string entirely from the encoded_strings and assigning into input_str"
+        )
+        input_str = new_str
+
+    return input_str
+
+
+def get_string(input_str: str, charset: str | None = None) -> str:
+    """Convert string to proper encoding with charset handling."""
+    if not input_str:
+        return input_str
+
+    if charset is None:
+        charset = "utf-8"
+
+    try:
+        return UnicodeDammit(input_str).unicode_markup.encode(charset).decode(charset)
+    except Exception:
+        try:
+            return str(make_header(decode_header(input_str)))
+        except Exception:
+            return decode_uni_string(input_str, input_str)
+
+
+def remove_child_info(file_path: str) -> str:
+    """Remove child info suffix from file path."""
+    if file_path.endswith("_True"):
+        return file_path.rstrip("_True")
+    return file_path.rstrip("_False")
+
+
+def create_dict_hash(input_dict: dict[str, Any]) -> str | None:
+    """Create a SHA256 hash of a dictionary."""
+    if not input_dict:
+        return None
+
+    try:
+        input_dict_str = json.dumps(input_dict, sort_keys=True)
+    except Exception as e:
+        logger.debug(f"Handled exception in create_dict_hash: {e}")
+        return None
+
+    try:
+        return hashlib.sha256(input_dict_str).hexdigest()  # type: ignore[arg-type]
+    except TypeError:
+        return hashlib.sha256(input_dict_str.encode("UTF-8")).hexdigest()

soar_sdk/input_spec.py

@@ -1,7 +1,8 @@
-from uuid import uuid4
-from pydantic import BaseModel, Field, field_validator, ConfigDict
-from typing import Literal, Any
 import random
+from typing import Any, Literal
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
 def id_factory() -> int:

soar_sdk/logging.py

@@ -1,11 +1,12 @@
 import logging
-from soar_sdk.colors import ANSIColor
+from typing import Any
 
-from soar_sdk.shims.phantom.install_info import is_soar_available, get_product_version
-from soar_sdk.shims.phantom.ph_ipc import ph_ipc
 from packaging.version import Version
-from typing import Any
+
+from soar_sdk.colors import ANSIColor
 from soar_sdk.compat import remove_when_soar_newer_than
+from soar_sdk.shims.phantom.install_info import get_product_version, is_soar_available
+from soar_sdk.shims.phantom.ph_ipc import ph_ipc
 
 PROGRESS_LEVEL = 25
 logging.addLevelName(PROGRESS_LEVEL, "PROGRESS")

soar_sdk/meta/actions.py

@@ -1,10 +1,10 @@
-from typing import Any, Type, Callable  # noqa: UP035
+from typing import Any, Callable, Type  # noqa: UP035
 
 from pydantic import BaseModel, Field
 
-from soar_sdk.cli.manifests.serializers import ParamsSerializer, OutputsSerializer
-from soar_sdk.params import Params
 from soar_sdk.action_results import ActionOutput
+from soar_sdk.cli.manifests.serializers import OutputsSerializer, ParamsSerializer
+from soar_sdk.params import Params
 
 
 class ActionMeta(BaseModel):

soar_sdk/meta/app.py

@@ -45,6 +45,7 @@ class AppMeta(BaseModel):
     pip314_dependencies: DependencyList = Field(default_factory=DependencyList)
 
     webhook: WebhookMeta | None = None
+    supports_es_polling: bool = False
 
     @field_validator("python_version", mode="before")
     @classmethod

soar_sdk/meta/dependencies.py

@@ -1,24 +1,21 @@
 import functools
+import hashlib
 import io
 import os
-from pathlib import Path
 import subprocess
 import tarfile
+from collections.abc import AsyncGenerator, Mapping, Sequence
+from logging import getLogger
+from pathlib import Path
 from tempfile import TemporaryDirectory
-import build
-
 from typing import ClassVar
-from collections.abc import Mapping, Sequence, AsyncGenerator
-from pydantic import BaseModel, Field
-
-from logging import getLogger
 
+import build
 import httpx
-import hashlib
+from pydantic import BaseModel, Field
 
 from soar_sdk.compat import remove_when_soar_newer_than
 
-
 logger = getLogger(__name__)
 
 # These dependencies are provided by the Python runner,
@@ -55,7 +52,8 @@ remove_when_soar_newer_than(
     "If the Splunk SDK is available as a wheel now, remove it, and remove all of the code for building wheels from source.",
 )
 DEPENDENCIES_TO_BUILD = {
-    "splunk_sdk",  # https://github.com/splunk/splunk-sdk-python/pull/656
+    "splunk_sdk",  # https://github.com/splunk/splunk-sdk-python/pull/656,
+    "splunk_soar_sdk",  # Useful to build from source when developing the SDK
 }
 
 
@@ -192,6 +190,23 @@ class UvSourceDistribution(BaseModel):
                 return Path(wheel_path).name, f.read()
 
 
+class UvSourceDirectory(BaseModel):
+    """Represents a Python dependency to be built from a source directory on the local filesystem."""
+
+    directory: str
+
+    def build(self) -> tuple[str, bytes]:
+        """Build a wheel from a local source directory."""
+        with TemporaryDirectory() as build_dir:
+            builder = build.ProjectBuilder(
+                self.directory,
+                runner=UvSourceDistribution._builder_runner,
+            )
+            wheel_path = builder.build("wheel", build_dir)
+            with open(wheel_path, "rb") as f:
+                return Path(wheel_path).name, f.read()
+
+
 class DependencyWheel(BaseModel):
     """Represents a Python package dependency with all the information required to fetch its wheel(s) from the CDN."""
 
@@ -202,6 +217,7 @@ class DependencyWheel(BaseModel):
     wheel: UvWheel | None = Field(exclude=True, default=None)
     wheel_aarch64: UvWheel | None = Field(exclude=True, default=None)
     sdist: UvSourceDistribution | None = Field(exclude=True, default=None)
+    source_dir: UvSourceDirectory | None = Field(exclude=True, default=None)
 
     async def collect_wheels(self) -> AsyncGenerator[tuple[str, bytes]]:
         """Collect a list of wheel files to fetch for this dependency across all platforms."""
@@ -211,6 +227,12 @@ class DependencyWheel(BaseModel):
             yield (f"wheels/shared/{wheel_name}", wheel_bytes)
             return
 
+        if self.wheel is None and self.source_dir is not None:
+            logger.info(f"Building local sources for {self.input_file}")
+            wheel_name, wheel_bytes = self.source_dir.build()
+            yield (f"wheels/shared/{wheel_name}", wheel_bytes)
+            return
+
         if self.wheel is None:
             raise ValueError(
                 f"Could not find a suitable wheel or source distribution for {self.module} in uv.lock"
@@ -250,6 +272,13 @@ class UvDependency(BaseModel):
     name: str
 
 
+class UvSource(BaseModel):
+    """Represents the source of a Python package in the uv lock."""
+
+    registry: str | None = None
+    directory: str | None = None
+
+
 class UvPackage(BaseModel):
     """Represents a Python package loaded from the uv lock."""
 
@@ -261,6 +290,7 @@ class UvPackage(BaseModel):
     )
     wheels: list[UvWheel] = []
     sdist: UvSourceDistribution | None = None
+    source: UvSource
 
     def _find_wheel(
         self,
@@ -369,6 +399,12 @@ class UvPackage(BaseModel):
         ):
             wheel.sdist = self.sdist
 
+        if (
+            self.source.directory is not None
+            and UvLock.normalize_package_name(self.name) in DEPENDENCIES_TO_BUILD
+        ):
+            wheel.source_dir = UvSourceDirectory(directory=self.source.directory)
+
         try:
             wheel_x86_64 = self._find_wheel(
                 abi_precedence, python_precedence, self.platform_precedence_x86_64
@@ -376,7 +412,7 @@ class UvPackage(BaseModel):
             wheel.input_file = f"{wheel_x86_64.basename}.whl"
             wheel.wheel = wheel_x86_64
         except FileNotFoundError as e:
-            if wheel.sdist is None:
+            if wheel.sdist is None and wheel.source_dir is None:
                 raise FileNotFoundError(
                     f"Could not find a suitable x86_64 wheel or source distribution for {self.name}"
                 ) from e

soar_sdk/meta/webhooks.py

@@ -1,6 +1,7 @@
-from pydantic import BaseModel, Field, field_validator
 from ipaddress import ip_network
 
+from pydantic import BaseModel, Field, field_validator
+
 
 class WebhookRouteMeta(BaseModel):
     """Metadata for a webhook route, including the handler function and its properties."""