iocflow 0.1.0__py3-none-any.whl

iocflow/__init__.py

@@ -0,0 +1,63 @@
+"""iocflow — an IOC-lifecycle toolkit.
+
+Layer 1 is threat-entity extraction: pull IPs, domains, URLs, filenames,
+hashes, CVEs, emails, MITRE technique IDs, threat actors, and malware families
+out of unstructured text. The extracted :class:`ExtractedEntities` is the input
+type that later layers (enrichment, AI commentary, hunt generation, blocking)
+build on.
+
+Quick start::
+
+    from iocflow import extract
+
+    entities = extract("APT28 used 185.220.101.5 and evil[.]example[.]com")
+    print(entities.summary())
+    for indicator in entities.iter_indicators():
+        print(indicator.kind, indicator.value)
+"""
+from iocflow.extract import extract
+from iocflow.extractors import (
+    extract_cves,
+    extract_domains,
+    extract_emails,
+    extract_filenames,
+    extract_hashes,
+    extract_ips,
+    extract_malware_families,
+    extract_mitre_procedures,
+    extract_mitre_techniques,
+    extract_threat_actors,
+    extract_urls,
+)
+from iocflow.models import ExtractedEntities, Indicator, ThreatActor
+from iocflow.providers import ActorAliases, MalwareNames
+from iocflow.refang import refang_text
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Orchestrator
+    "extract",
+    # Result types
+    "ExtractedEntities",
+    "Indicator",
+    "ThreatActor",
+    # Pluggable sources
+    "ActorAliases",
+    "MalwareNames",
+    # Individual extractors
+    "extract_ips",
+    "extract_domains",
+    "extract_urls",
+    "extract_emails",
+    "extract_filenames",
+    "extract_hashes",
+    "extract_cves",
+    "extract_mitre_techniques",
+    "extract_mitre_procedures",
+    "extract_threat_actors",
+    "extract_malware_families",
+    # Utilities
+    "refang_text",
+    "__version__",
+]

iocflow/allowlists.py

@@ -0,0 +1,177 @@
+"""Allowlists and blocklists used to suppress false positives.
+
+These are deliberately exported as plain module-level sets so callers can
+extend or override them, e.g.::
+
+    from iocflow import allowlists
+    allowlists.BENIGN_DOMAINS.add("corp.example.net")
+"""
+from __future__ import annotations
+
+# Benign domains to exclude during extraction. The IOC-hunt flow bypasses
+# reputation filtering, so this list is intentionally broad: test/placeholder
+# domains, package registries, security-vendor sites, threat-intel references,
+# and major cloud/CDN infrastructure that routinely appears in reports.
+BENIGN_DOMAINS = {
+    # Test / placeholder domains
+    "example.com", "example.org", "example.net", "localhost", "test.com",
+    "internal.local",
+    # Common email providers (email addresses still extracted, just not as domain IOCs)
+    "gmail.com", "hotmail.com", "yahoo.com", "outlook.com", "live.com",
+    # Package registries — legitimate infrastructure, not IOCs
+    "npmjs.org", "registry.npmjs.org", "yarn.npmjs.org",
+    "yarnpkg.com", "registry.yarnpkg.com",
+    "github.com", "raw.githubusercontent.com", "gist.github.com",
+    "pypi.org", "files.pythonhosted.org",
+    "rubygems.org", "nuget.org", "crates.io",
+    "packagist.org", "mvnrepository.com", "maven.org",
+    "docker.io", "docker.com", "hub.docker.com",
+    # Cybersecurity vendors — appear as references in reports, not IOCs
+    "paloaltonetworks.com", "unit42.paloaltonetworks.com",
+    "crowdstrike.com", "falcon.crowdstrike.com",
+    "mandiant.com", "cloud.google.com",
+    "microsoft.com", "learn.microsoft.com", "security.microsoft.com",
+    "cisco.com", "talosintelligence.com",
+    "fortinet.com", "fortiguard.com",
+    "sentinelone.com", "sentinellabs.com",
+    "trendmicro.com",
+    "sophos.com", "news.sophos.com",
+    "symantec.com", "broadcom.com",
+    "mcafee.com", "trellix.com",
+    "fireeye.com",
+    "elastic.co", "elastic.github.io",
+    "zscaler.com",
+    "proofpoint.com",
+    "checkpoint.com", "research.checkpoint.com",
+    "recordedfuture.com",
+    "sekoia.io",
+    "group-ib.com",
+    "kaspersky.com", "securelist.com",
+    "eset.com", "welivesecurity.com",
+    "bitdefender.com",
+    "malwarebytes.com",
+    "cybereason.com",
+    "rapid7.com",
+    "qualys.com",
+    "tenable.com",
+    "dragos.com",
+    "volexity.com",
+    "huntress.com",
+    "infoblox.com",
+    # Threat-intel / research references
+    "mitre.org", "attack.mitre.org", "cve.mitre.org",
+    "krebsonsecurity.com",
+    "bleepingcomputer.com",
+    "thehackernews.com",
+    "therecord.media",
+    "darkreading.com",
+    "securityweek.com",
+    "threatpost.com",
+    "cyberscoop.com",
+    "schneier.com",
+    "nist.gov", "nvd.nist.gov",
+    "cisa.gov", "us-cert.cisa.gov",
+    "cert.org",
+    "virustotal.com",
+    "shodan.io",
+    "abuse.ch", "bazaar.abuse.ch", "urlhaus.abuse.ch", "threatfox.abuse.ch",
+    "otx.alienvault.com", "alienvault.com",
+    "hybrid-analysis.com",
+    "any.run", "app.any.run",
+    "joesandbox.com", "joesecurity.org",
+    "urlscan.io",
+    "whois.domaintools.com", "domaintools.com",
+    "abuseipdb.com",
+    # Cloud / CDN infrastructure
+    "amazonaws.com", "azure.com", "azureedge.net",
+    "cloudflare.com", "cloudfront.net",
+    "akamai.com", "akamaitechnologies.com",
+    "googleapis.com",
+    "windows.net", "office365.com", "office.com",
+    "sharepoint.com", "onedrive.com",
+    "google.com", "gstatic.com",
+    "linkedin.com", "twitter.com", "x.com",
+    "wikipedia.org", "medium.com",
+}
+
+# Package-registry hosts: the host is benign infrastructure, but a *path* under
+# it (registry.npmjs.org/<pkg>) can name a malicious package, so URL paths on
+# these hosts are kept even though the bare host is in BENIGN_DOMAINS.
+PACKAGE_REGISTRY_HOSTS = {
+    "npmjs.org", "registry.npmjs.org", "yarn.npmjs.org",
+    "yarnpkg.com", "registry.yarnpkg.com",
+    "pypi.org", "files.pythonhosted.org",
+    "rubygems.org", "nuget.org", "crates.io",
+    "packagist.org", "mvnrepository.com", "maven.org",
+}
+
+# Known benign IPs to exclude (loopback, broadcast, public resolvers).
+BENIGN_IPS = {
+    "127.0.0.1", "0.0.0.0", "255.255.255.255",
+    "8.8.8.8", "8.8.4.4",  # Google DNS
+    "1.1.1.1", "1.0.0.1",  # Cloudflare DNS
+}
+
+# Extensions that are also valid TLDs. A bare "word.ext" with one of these is
+# usually a filename (install.sh) rather than a domain (openclaw.ai), so it is
+# only accepted as a domain when it doesn't look like a common filename.
+FILE_EXTENSION_TLDS = {
+    "sh", "py", "pl", "rs", "ps", "cc", "md", "so", "la", "do", "to",
+    "ai", "st", "fm", "am", "dj", "gs", "ms", "lk", "im", "ws", "nu", "tk",
+}
+
+# Common filenames that collide with file-extension TLDs (install.sh, setup.py).
+COMMON_FILENAME_STEMS = {
+    "install", "setup", "script", "run", "start", "init",
+    "main", "index", "test", "build", "deploy", "config",
+}
+
+# Common English words that happen to be MITRE malware/tool names.
+MALWARE_BLOCKLIST = {
+    "anchor", "chaos", "empire", "expand", "flame", "havoc",
+    "james", "kevin", "milan", "mango", "meteor", "net", "ninja",
+    "ping", "rover", "royal", "ruler", "shark", "snake", "spark",
+    "solar", "page", "play",
+}
+
+# LOLBins — legitimate Windows/system utilities MITRE tracks as "tools".
+SYSTEM_TOOL_BLOCKLIST = {
+    "arp", "at", "attrib", "bitsadmin", "certutil", "cipher.exe",
+    "cmd", "dsquery", "esentutl", "forfiles", "ftp", "ifconfig",
+    "ipconfig", "nbtstat", "nbtscan", "net", "netsh", "netstat",
+    "nltest", "ping", "psexec", "pwdump", "rclone", "reg", "route",
+    "schtasks", "sdelete", "systeminfo", "tasklist", "tor", "wevtutil",
+    "connectwise", "quick assist",
+}
+
+# Well-known threat-actor and ransomware names matched by exact word boundary.
+WELL_KNOWN_ACTORS = [
+    # APT groups
+    "Lazarus", "Lazarus Group",
+    "Fancy Bear", "Cozy Bear",
+    "Sandworm", "Turla",
+    "Kimsuky", "Charming Kitten",
+    "OceanLotus", "Ocean Lotus",
+    "Equation Group",
+    "Scattered Spider",
+    "Nobelium", "Midnight Blizzard",
+    "Volt Typhoon", "Salt Typhoon",
+    # Ransomware families
+    "ALPHV", "BlackCat",
+    "LockBit", "Conti", "REvil",
+    "CrazyHunter", "Akira", "Play",
+    "Royal", "Black Basta", "BlackBasta",
+    "Cl0p", "Clop", "Cuba", "Hive",
+    "Medusa", "Rhysida", "BianLian",
+    "NoEscape", "Cactus", "Hunters International",
+    "Qilin", "INC Ransom", "RansomHub",
+    "DragonForce", "Fog", "Lynx",
+]
+
+# Capitalized words that precede "ransomware" but are not actor names.
+RANSOMWARE_FALSE_POSITIVES = {
+    "The", "This", "That", "New", "Old", "Some", "Any", "Each", "Our", "Their",
+    "Executed", "Propagated", "Deployed", "Distributed", "Disguised", "Downloaded",
+    "Encrypted", "Delivered", "Launched", "Installed", "Targeted", "Modified",
+    "Prince",  # Usually "fork of Prince ransomware" — context, not actor
+}

iocflow/cli.py

@@ -0,0 +1,62 @@
+"""Command-line entry point: ``python -m iocflow`` (or the ``iocflow`` script)."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from iocflow.extract import extract
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="iocflow",
+        description="Extract threat indicators (IOCs) from text.",
+    )
+    parser.add_argument(
+        "text",
+        nargs="*",
+        help="Text to extract from. If omitted, reads from stdin.",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Emit the full result as JSON instead of a human summary.",
+    )
+    parser.add_argument(
+        "--no-refang",
+        action="store_true",
+        help="Do not re-fang defanged IOCs before extracting.",
+    )
+    parser.add_argument(
+        "--mitre",
+        action="store_true",
+        help="Load MITRE malware names for family extraction (needs iocflow[mitre]).",
+    )
+    args = parser.parse_args(argv)
+
+    text = " ".join(args.text) if args.text else sys.stdin.read()
+
+    malware_names = None
+    if args.mitre:
+        try:
+            from iocflow.mitre import mitre_malware_names
+
+            malware_names = mitre_malware_names()
+        except ImportError:
+            parser.error("--mitre requires the extra: pip install 'iocflow[mitre]'")
+
+    entities = extract(text, malware_names=malware_names, refang=not args.no_refang)
+
+    if args.json:
+        json.dump(entities.to_dict(), sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    else:
+        print(entities.summary())
+        for indicator in entities.iter_indicators():
+            print(f"  {indicator.kind:16} {indicator.value}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

iocflow/extract.py

@@ -0,0 +1,99 @@
+"""The :func:`extract` orchestrator — run every extractor over a piece of text."""
+from __future__ import annotations
+
+import logging
+import re
+from typing import Optional
+
+from iocflow.extractors.actors import extract_malware_families, extract_threat_actors
+from iocflow.extractors.contacts import extract_emails
+from iocflow.extractors.files import extract_filenames, extract_hashes
+from iocflow.extractors.network import extract_domains, extract_ips, extract_urls
+from iocflow.extractors.vulns import extract_cves, extract_mitre_techniques
+from iocflow.models import ExtractedEntities, ThreatActor
+from iocflow.providers import ActorAliases, MalwareNames
+from iocflow.refang import refang_text
+
+logger = logging.getLogger(__name__)
+
+_HTML_TAG = re.compile(r"<[^>]+>")
+_WHITESPACE = re.compile(r"\s+")
+
+
+def extract(
+    text: str,
+    *,
+    actor_aliases: Optional[ActorAliases] = None,
+    malware_names: Optional[MalwareNames] = None,
+    refang: bool = True,
+) -> ExtractedEntities:
+    """Extract all entity types from ``text``.
+
+    Args:
+        text: The text to extract from.
+        actor_aliases: Optional known-actor names + alias index. When given,
+            actor names are also matched against this set and enriched with
+            ``common_name`` / ``region`` / ``all_names``.
+        malware_names: Optional malware/tool name set. Required for
+            ``malware_families`` to be populated.
+        refang: Whether to re-fang defanged IOCs (``[.]`` -> ``.`` etc.) first.
+
+    Returns:
+        An :class:`~iocflow.models.ExtractedEntities`.
+    """
+    if not text:
+        return ExtractedEntities()
+
+    # Strip HTML and collapse whitespace.
+    clean = _HTML_TAG.sub(" ", text)
+    clean = _WHITESPACE.sub(" ", clean)
+    if refang:
+        clean = refang_text(clean)
+
+    raw_actors = extract_threat_actors(clean, actor_aliases)
+    enriched = _enrich_actors(raw_actors, actor_aliases)
+
+    urls = extract_urls(clean)
+    entities = ExtractedEntities(
+        ips=extract_ips(clean),
+        domains=extract_domains(clean),
+        urls=urls,
+        filenames=extract_filenames(clean, urls=urls),
+        hashes=extract_hashes(clean),
+        cves=extract_cves(clean),
+        emails=extract_emails(clean),
+        threat_actors=raw_actors,
+        threat_actors_enriched=enriched,
+        malware_families=extract_malware_families(clean, malware_names),
+        mitre_techniques=extract_mitre_techniques(clean),
+    )
+
+    if not entities.is_empty():
+        logger.info("Extracted entities: %s", entities.summary())
+    return entities
+
+
+def _enrich_actors(
+    raw_actors: list, actor_aliases: Optional[ActorAliases]
+) -> list:
+    """Map raw actor names to :class:`ThreatActor`, de-duplicating by common name."""
+    enriched = []
+    seen_common = set()
+    for name in raw_actors:
+        info = actor_aliases.lookup(name) if actor_aliases else None
+        if info:
+            common = info.get("common_name", name)
+            if common.lower() in seen_common:
+                continue
+            seen_common.add(common.lower())
+            enriched.append(
+                ThreatActor(
+                    name=name,
+                    common_name=common,
+                    region=info.get("region", ""),
+                    all_names=info.get("all_names", []),
+                )
+            )
+        else:
+            enriched.append(ThreatActor(name=name, common_name=name))
+    return enriched

iocflow/extractors/__init__.py

@@ -0,0 +1,35 @@
+"""Individual, composable extraction functions.
+
+Each function takes text and returns a list/dict of one entity type. They are
+re-exported from the top-level :mod:`iocflow` package for convenience.
+"""
+from iocflow.extractors.actors import (
+    extract_malware_families,
+    extract_threat_actors,
+)
+from iocflow.extractors.contacts import extract_emails
+from iocflow.extractors.files import extract_filenames, extract_hashes
+from iocflow.extractors.network import (
+    extract_domains,
+    extract_ips,
+    extract_urls,
+)
+from iocflow.extractors.vulns import (
+    extract_cves,
+    extract_mitre_procedures,
+    extract_mitre_techniques,
+)
+
+__all__ = [
+    "extract_ips",
+    "extract_domains",
+    "extract_urls",
+    "extract_emails",
+    "extract_filenames",
+    "extract_hashes",
+    "extract_cves",
+    "extract_mitre_techniques",
+    "extract_mitre_procedures",
+    "extract_threat_actors",
+    "extract_malware_families",
+]

iocflow/extractors/actors.py

@@ -0,0 +1,107 @@
+"""Threat-actor and malware-family extraction.
+
+Actor extraction works with zero external data via patterns and a curated
+well-known list. Pass an :class:`~iocflow.providers.ActorAliases` to also match
+a custom name set, and a :class:`~iocflow.providers.MalwareNames` to enable
+malware-family extraction.
+"""
+from __future__ import annotations
+
+import re
+from typing import List, Optional
+
+from iocflow.allowlists import (
+    MALWARE_BLOCKLIST,
+    RANSOMWARE_FALSE_POSITIVES,
+    SYSTEM_TOOL_BLOCKLIST,
+    WELL_KNOWN_ACTORS,
+)
+from iocflow.providers import ActorAliases, MalwareNames
+
+# Structured actor designators: APT28, APT-28, UNC2452, FIN7, TA505, DEV-0537,
+# STORM-0558.
+_ACTOR_PATTERNS = [
+    re.compile(r"\bAPT[-]?\d+\b", re.IGNORECASE),
+    re.compile(r"\bUNC\d+\b", re.IGNORECASE),
+    re.compile(r"\bFIN\d+\b", re.IGNORECASE),
+    re.compile(r"\bTA\d+\b", re.IGNORECASE),
+    re.compile(r"\bDEV-\d+\b", re.IGNORECASE),
+    re.compile(r"\bSTORM-\d+\b", re.IGNORECASE),
+]
+
+# "<ProperNoun> ransomware" — case-sensitive so "go-based ransomware" misses.
+_RANSOMWARE = re.compile(r"\b([A-Z][a-z]+(?:[A-Z][a-z0-9]*)*)\s+ransomware\b")
+
+# Real APT names that are common English words and cause too many false hits.
+_ACTOR_NAME_BLOCKLIST = {"lead"}
+
+
+def extract_threat_actors(text: str, known: Optional[ActorAliases] = None) -> List[str]:
+    """Extract threat-actor names.
+
+    Strategies, in order:
+
+    1. Exact-match a caller-supplied known-name set (if ``known`` is given).
+    2. Match APT/UNC/FIN/TA/DEV/STORM designators.
+    3. Match a curated list of well-known actor and ransomware names.
+    4. Catch the ``"<Name> ransomware"`` pattern.
+    """
+    actors = set()
+
+    if known is not None:
+        for name in known.known_names:
+            if len(name) <= 3 or name.lower() in _ACTOR_NAME_BLOCKLIST:
+                continue
+            m = re.search(r"\b" + re.escape(name) + r"\b", text, re.IGNORECASE)
+            if m:
+                actors.add(m.group())
+
+    for pattern in _ACTOR_PATTERNS:
+        actors.update(m.upper() for m in pattern.findall(text))
+
+    for actor in WELL_KNOWN_ACTORS:
+        m = re.search(r"\b" + re.escape(actor) + r"\b", text, re.IGNORECASE)
+        if m:
+            actors.add(m.group())
+
+    for m in _RANSOMWARE.finditer(text):
+        name = m.group(1)
+        if name not in RANSOMWARE_FALSE_POSITIVES and len(name) >= 4:
+            actors.add(name)
+
+    return list(actors)
+
+
+def extract_malware_families(text: str, malware: Optional[MalwareNames] = None) -> List[str]:
+    """Extract malware-family names by matching a caller-supplied name set.
+
+    Returns ``[]`` when no :class:`~iocflow.providers.MalwareNames` is given.
+    Three-layer false-positive defense:
+
+    1. Skip names of 3 characters or fewer (``at``, ``cmd``, ``ftp``).
+    2. Case-sensitive matching for single-word names; case-insensitive for
+       multi-word names.
+    3. Blocklist common English words and LOLBins.
+
+    When an alias matches, the result is normalized to its canonical name.
+    """
+    if malware is None or not malware.names:
+        return []
+
+    matched = set()
+    for name in malware.names:
+        if len(name) <= 3:
+            continue
+        name_lower = name.lower()
+        if name_lower in MALWARE_BLOCKLIST or name_lower in SYSTEM_TOOL_BLOCKLIST:
+            continue
+
+        flags = re.IGNORECASE if " " in name else 0
+        if re.search(r"\b" + re.escape(name) + r"\b", text, flags):
+            canonical = malware.alias_map.get(name_lower, name)
+            canon_lower = canonical.lower()
+            if canon_lower in MALWARE_BLOCKLIST or canon_lower in SYSTEM_TOOL_BLOCKLIST:
+                continue
+            matched.add(canonical)
+
+    return sorted(matched)

iocflow/extractors/contacts.py

@@ -0,0 +1,15 @@
+"""Contact indicators: email addresses."""
+from __future__ import annotations
+
+import re
+from typing import List
+
+_EMAIL = re.compile(r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b", re.IGNORECASE)
+
+MAX_EMAILS = 20
+
+
+def extract_emails(text: str) -> List[str]:
+    """Extract email addresses, lowercased and de-duplicated."""
+    matches = _EMAIL.findall(text)
+    return list(dict.fromkeys(e.lower() for e in matches))[:MAX_EMAILS]

iocflow/extractors/files.py

@@ -0,0 +1,84 @@
+"""File indicators: suspicious filenames and cryptographic hashes."""
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional
+
+# Extensions that indicate a potentially malicious file. ``.com`` is excluded
+# because it collides with domain names (github.com).
+SUSPICIOUS_EXTENSIONS = {
+    # Scripts
+    "ps1", "sh", "bat", "cmd", "vbs", "vbe", "js", "jse", "wsf", "wsh",
+    # Executables (no .com — avoids domain false positives)
+    "exe", "dll", "msi", "scr", "pif",
+    # Documents with macros
+    "docm", "xlsm", "pptm", "dotm", "xltm",
+    # Archives (can carry malware)
+    "iso", "img", "vhd", "vhdx",
+    # Other
+    "hta", "lnk", "jar", "msc",
+}
+
+_EXT_ALTERNATION = "|".join(re.escape(ext) for ext in SUSPICIOUS_EXTENSIONS)
+_FILENAME = re.compile(rf"\b([a-zA-Z0-9_\-\.]+\.(?:{_EXT_ALTERNATION}))\b", re.IGNORECASE)
+
+MAX_FILENAMES = 20
+
+
+def extract_filenames(text: str, urls: Optional[List[str]] = None) -> List[str]:
+    """Extract suspicious script/executable filenames from text and URLs."""
+    filenames: List[str] = []
+    seen = set()
+
+    for match in _FILENAME.finditer(text):
+        filename = match.group(1)
+        if filename.lower() not in seen:
+            filenames.append(filename)
+            seen.add(filename.lower())
+
+    if urls:
+        for url in urls:
+            path = url.split("/")[-1]
+            if path and "." in path:
+                ext = path.rsplit(".", 1)[-1].lower()
+                if ext in SUSPICIOUS_EXTENSIONS and path.lower() not in seen:
+                    filenames.append(path)
+                    seen.add(path.lower())
+
+    return filenames[:MAX_FILENAMES]
+
+
+_SHA256 = re.compile(r"\b[a-fA-F0-9]{64}\b")
+_SHA1 = re.compile(r"\b[a-fA-F0-9]{40}\b")
+_MD5 = re.compile(r"\b[a-fA-F0-9]{32}\b")
+
+
+def extract_hashes(text: str) -> Dict[str, List[str]]:
+    """Extract MD5, SHA1, and SHA256 hashes.
+
+    Longer hashes are matched first; shorter patterns that are merely a prefix
+    of an already-matched longer hash are dropped, so a SHA256 isn't also
+    reported as a SHA1 and an MD5.
+    """
+    hashes: Dict[str, List[str]] = {"md5": [], "sha1": [], "sha256": []}
+
+    hashes["sha256"] = list(dict.fromkeys(h.lower() for h in _SHA256.findall(text)))
+
+    sha256_prefixes_40 = {h[:40] for h in hashes["sha256"]}
+    hashes["sha1"] = list(
+        dict.fromkeys(
+            h.lower() for h in _SHA1.findall(text) if h.lower() not in sha256_prefixes_40
+        )
+    )
+
+    sha1_prefixes_32 = {h[:32] for h in hashes["sha1"]}
+    sha256_prefixes_32 = {h[:32] for h in hashes["sha256"]}
+    hashes["md5"] = list(
+        dict.fromkeys(
+            h.lower()
+            for h in _MD5.findall(text)
+            if h.lower() not in sha1_prefixes_32 and h.lower() not in sha256_prefixes_32
+        )
+    )
+
+    return hashes