iocscrape 0.2.2__tar.gz → 0.2.3__tar.gz

{iocscrape-0.2.2 → iocscrape-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iocscrape
-Version: 0.2.2
+Version: 0.2.3
 Summary: CTI tool to extract IOCs from CTI reports (URLs or files), and write them to an output file. Low-confidence items are grouped at the end.
 Author-email: Faisal Albuloushi <SQL@hotmail.co.uk>
 License: The MIT License (MIT)

{iocscrape-0.2.2 → iocscrape-0.2.3}/iocscrape/cli.py

@@ -24,7 +24,7 @@ from publicsuffix2 import get_sld
 # ========= #
 
 APP_NAME = "iocscrape"
-APP_VERSION = "0.2.2"
+APP_VERSION = "0.2.3"
 PROJECT_URL = "https://github.com/fwalbuloushi/iocscrape"
 APP_DESC = "CTI tool to extract IOCs from CTI reports (URLs or files), and write them to an output file. Low-confidence items are grouped at the end."
 
@@ -502,13 +502,15 @@ class WarninglistIndex:
         if d in self._exact_map:
             return self._exact_map[d]
 
-        # 2. Check parent domains (suffix match)
+        # 2. Check parent domains (suffix match) — skip bare TLDs
         parts = d.split(".")
         for i in range(1, len(parts)):
             parent = ".".join(parts[i:])
+            if "." not in parent:
+                continue
             if parent in self._exact_map:
                 return self._exact_map[parent]
-                
+
         return None
 
     def match_url(self, url: str) -> Optional[str]:

{iocscrape-0.2.2 → iocscrape-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "iocscrape"
-version = "0.2.2"
+version = "0.2.3"
 description = "CTI tool to extract IOCs from CTI reports (URLs or files), and write them to an output file. Low-confidence items are grouped at the end."
 readme = "README.md"
 requires-python = ">=3.10"

iocscrape-0.2.3/tests/test_deobfuscate.py

@@ -0,0 +1,34 @@
+from iocscrape.cli import deobfuscate_text
+
+
+def test_hxxps():
+    assert deobfuscate_text("hxxps://evil.com") == "https://evil.com"
+
+def test_hxxp():
+    assert deobfuscate_text("hxxp://evil.com") == "http://evil.com"
+
+def test_dot_bracket():
+    assert deobfuscate_text("evil[.]com") == "evil.com"
+
+def test_dot_paren():
+    assert deobfuscate_text("evil(.)com") == "evil.com"
+
+def test_dot_brace():
+    assert deobfuscate_text("evil{.}com") == "evil.com"
+
+def test_dot_word():
+    assert deobfuscate_text("evil dot com") == "evil.com"
+
+def test_dot_word_with_spaces():
+    assert deobfuscate_text("evil (dot) com") == "evil.com"
+
+def test_colon_bracket():
+    assert deobfuscate_text("hxxps[://]evil.com") == "https://evil.com"
+
+def test_mixed():
+    result = deobfuscate_text("hxxps://evil[.]com/path")
+    assert result == "https://evil.com/path"
+
+def test_no_change():
+    original = "https://legitimate.com"
+    assert deobfuscate_text(original) == original

iocscrape-0.2.3/tests/test_extract.py

@@ -0,0 +1,53 @@
+from iocscrape.cli import extract_iocs
+
+
+def test_url():
+    iocs = extract_iocs("Payload was downloaded from https://malware.example.com/payload")
+    assert "https://malware.example.com/payload" in iocs["url"]
+
+def test_domain():
+    iocs = extract_iocs("The threat actor used evil.example.com as a C2 server")
+    assert "evil.example.com" in iocs["domain"]
+
+def test_ipv4():
+    iocs = extract_iocs("Beacon traffic was observed to 192.168.1.100 on port 443")
+    assert "192.168.1.100" in iocs["ipv4"]
+
+def test_ipv6():
+    iocs = extract_iocs("C2 communication detected to 2001:db8::1 over port 8443")
+    assert "2001:db8::1" in iocs["ipv6"]
+
+def test_email():
+    iocs = extract_iocs("Spear-phishing email was sent from attacker@evil.com")
+    assert "attacker@evil.com" in iocs["email"]
+
+def test_md5():
+    iocs = extract_iocs("Malware sample identified with MD5: d41d8cd98f00b204e9800998ecf8427e")
+    assert "d41d8cd98f00b204e9800998ecf8427e" in iocs["md5"]
+
+def test_sha1():
+    iocs = extract_iocs("Dropper hash SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709")
+    assert "da39a3ee5e6b4b0d3255bfef95601890afd80709" in iocs["sha1"]
+
+def test_sha256():
+    iocs = extract_iocs("Ransomware binary SHA256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+    assert "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" in iocs["sha256"]
+
+def test_cve():
+    iocs = extract_iocs("The exploit targets CVE-2021-44228 to achieve remote code execution")
+    assert "CVE-2021-44228" in iocs["cve"]
+
+def test_sha256_not_in_sha1_or_md5():
+    """A 64-char hash should only appear in sha256, not sha1 or md5."""
+    h = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    iocs = extract_iocs(h)
+    assert h in iocs["sha256"]
+    assert h not in iocs["sha1"]
+    assert h not in iocs["md5"]
+
+def test_sha1_not_in_md5():
+    """A 40-char hash should only appear in sha1, not md5."""
+    h = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    iocs = extract_iocs(h)
+    assert h in iocs["sha1"]
+    assert h not in iocs["md5"]

iocscrape-0.2.3/tests/test_low_confidence.py

@@ -0,0 +1,66 @@
+from iocscrape.cli import compute_low_confidence, load_warninglists, WarninglistIndex
+
+
+def _wl():
+    return load_warninglists()
+
+
+def test_filename_like_domain_flagged():
+    iocs = {"domain": {"malware.png"}, "url": set(), "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, _wl())
+    values = [item.value for item in low]
+    assert "malware.png" in values
+
+def test_static_asset_url_flagged():
+    iocs = {"domain": set(), "url": {"https://cdn.example.com/style.css"}, "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, _wl())
+    values = [item.value for item in low]
+    assert "https://cdn.example.com/style.css" in values
+
+def test_private_ipv4_flagged():
+    iocs = {"domain": set(), "url": set(), "ipv4": {"10.0.0.1"}, "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, _wl())
+    values = [item.value for item in low]
+    assert "10.0.0.1" in values
+
+def test_loopback_ipv4_flagged():
+    iocs = {"domain": set(), "url": set(), "ipv4": {"127.0.0.1"}, "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, _wl())
+    values = [item.value for item in low]
+    assert "127.0.0.1" in values
+
+def test_warninglist_domain_is_flagged():
+    """A domain that is explicitly in the warninglist should be flagged as low-confidence."""
+    wl = WarninglistIndex()
+    wl._exact_map["google.com"] = "Test warninglist"
+    iocs = {"domain": {"google.com"}, "url": set(), "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, wl)
+    values = [item.value for item in low]
+    assert "google.com" in values
+
+def test_subdomain_of_warninglist_domain_is_flagged():
+    """A subdomain of a warninglist entry should be flagged via suffix matching."""
+    wl = WarninglistIndex()
+    wl._exact_map["mitre.org"] = "Test warninglist"
+    iocs = {"domain": {"sub.mitre.org"}, "url": set(), "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, wl)
+    values = [item.value for item in low]
+    assert "sub.mitre.org" in values
+
+def test_bare_tld_in_warninglist_does_not_flag_all_domains():
+    """A bare TLD entry in the warninglist (e.g. 'com') must not cause every
+    .com domain to be flagged as low-confidence via suffix matching."""
+    wl = WarninglistIndex()
+    wl._exact_map["com"] = "IANA TLDs"
+    iocs = {"domain": {"evil-c2.com"}, "url": set(), "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, wl)
+    values = [item.value for item in low]
+    assert "evil-c2.com" not in values
+
+def test_ioc_domain_not_false_positive():
+    """A real C2-style IOC domain whose parent is not in any warninglist
+    should stay high-confidence and not be moved to low-confidence."""
+    iocs = {"domain": {"c2-server.badactor-infra.com"}, "url": set(), "ipv4": set(), "ipv6": set(), "email": set(), "md5": set(), "sha1": set(), "sha256": set(), "cve": set()}
+    low = compute_low_confidence(iocs, _wl())
+    values = [item.value for item in low]
+    assert "c2-server.badactor-infra.com" not in values