polyswarm-api 4.5.0__tar.gz → 4.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {polyswarm_api-4.5.0/src/polyswarm_api.egg-info → polyswarm_api-4.7.0}/PKG-INFO +1 -1
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/pyproject.toml +2 -2
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/__init__.py +2 -1
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/aio/api.py +65 -12
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/api.py +87 -13
- polyswarm_api-4.7.0/src/polyswarm_api/refang.py +113 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/resources.py +7 -2
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0/src/polyswarm_api.egg-info}/PKG-INFO +1 -1
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api.egg-info/SOURCES.txt +1 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/LICENSE +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/README.md +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/setup.cfg +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/aio/__init__.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/aio/session.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/core.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/exceptions.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/session.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api/settings.py +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api.egg-info/dependency_links.txt +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api.egg-info/requires.txt +0 -0
- {polyswarm_api-4.5.0 → polyswarm_api-4.7.0}/src/polyswarm_api.egg-info/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "polyswarm_api"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.7.0"
|
|
8
8
|
description = "Client library to simplify interacting with the PolySwarm consumer API"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10,<4"
|
|
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
|
|
|
55
55
|
where = ["src"]
|
|
56
56
|
|
|
57
57
|
[tool.bumpversion]
|
|
58
|
-
current_version = "4.
|
|
58
|
+
current_version = "4.7.0"
|
|
59
59
|
commit = true
|
|
60
60
|
tag = false
|
|
61
61
|
sign_tags = true
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
|
|
2
|
-
__version__ = '4.
|
|
2
|
+
__version__ = '4.7.0'
|
|
3
3
|
__release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest'
|
|
4
4
|
|
|
5
5
|
from . import api
|
|
6
6
|
from . import exceptions
|
|
7
|
+
from . import refang
|
|
7
8
|
from .api import PolyswarmAPI
|
|
8
9
|
from .session import PolyswarmSession
|
|
9
10
|
from .aio import PolySwarmAsyncAPI, AsyncPolyswarmSession
|
|
@@ -17,7 +17,7 @@ import io
|
|
|
17
17
|
import logging
|
|
18
18
|
import time
|
|
19
19
|
|
|
20
|
-
from polyswarm_api import exceptions, resources, settings
|
|
20
|
+
from polyswarm_api import exceptions, refang, resources, settings
|
|
21
21
|
from polyswarm_api.core import PolyswarmRequest, _as_result_bound
|
|
22
22
|
|
|
23
23
|
from .session import AsyncPolyswarmSession
|
|
@@ -53,6 +53,7 @@ class PolySwarmAsyncAPI:
|
|
|
53
53
|
verify: bool = True,
|
|
54
54
|
*,
|
|
55
55
|
session: AsyncPolyswarmSession | None = None,
|
|
56
|
+
refang_iocs: bool = False,
|
|
56
57
|
**httpx_kwargs,
|
|
57
58
|
):
|
|
58
59
|
key_masked = '******' + (key[-4:] if key and len(key) > 16 else '')
|
|
@@ -64,6 +65,11 @@ class PolySwarmAsyncAPI:
|
|
|
64
65
|
self.community = community or settings.DEFAULT_COMMUNITY
|
|
65
66
|
self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
|
|
66
67
|
self.verify = verify
|
|
68
|
+
# Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
|
|
69
|
+
# before building a request. Opt-in: the default preserves the 4.5.0
|
|
70
|
+
# behaviour of sending every input verbatim. See ``polyswarm_api.refang``
|
|
71
|
+
# and ``_refang`` below for exactly which inputs are touched.
|
|
72
|
+
self.refang_iocs = refang_iocs
|
|
67
73
|
self._engines = None
|
|
68
74
|
# Either accept a pre-built session (customization point) or
|
|
69
75
|
# build the default from ``key``. Passing both is ambiguous.
|
|
@@ -88,6 +94,27 @@ class PolySwarmAsyncAPI:
|
|
|
88
94
|
attrs = f'uri={self.uri!r}, community={self.community!r}, timeout={self.timeout!r}'
|
|
89
95
|
return f'<{clsname}({attrs}) at 0x{id(self):x}>'
|
|
90
96
|
|
|
97
|
+
def _refang(self, value):
|
|
98
|
+
"""Refang one URL / domain / IP input when ``refang_iocs`` is on.
|
|
99
|
+
|
|
100
|
+
Applied only to arguments that name a network indicator (a URL to
|
|
101
|
+
search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
|
|
102
|
+
``ip`` / ``domain``, a known-host ``host``) — never to a free-form
|
|
103
|
+
metadata query or a hash.
|
|
104
|
+
Anything that is not a defanged network IoC is returned unchanged.
|
|
105
|
+
"""
|
|
106
|
+
if not self.refang_iocs:
|
|
107
|
+
return value
|
|
108
|
+
return refang.refang_ioc(value)
|
|
109
|
+
|
|
110
|
+
def _refang_all(self, values):
|
|
111
|
+
"""``_refang`` over a list argument; ``None`` / empty pass through."""
|
|
112
|
+
if not values or not self.refang_iocs:
|
|
113
|
+
return values
|
|
114
|
+
if isinstance(values, str):
|
|
115
|
+
return self._refang(values)
|
|
116
|
+
return [self._refang(v) for v in values]
|
|
117
|
+
|
|
91
118
|
async def aclose(self):
|
|
92
119
|
"""Close the underlying HTTP client."""
|
|
93
120
|
await self.session.aclose()
|
|
@@ -340,6 +367,7 @@ class PolySwarmAsyncAPI:
|
|
|
340
367
|
:param url: A url to be searched by exact match
|
|
341
368
|
:return: Generator of ArtifactInstance resources
|
|
342
369
|
"""
|
|
370
|
+
url = self._refang(url)
|
|
343
371
|
logger.info('Searching for url %s', url)
|
|
344
372
|
async for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
|
|
345
373
|
yield item
|
|
@@ -365,6 +393,7 @@ class PolySwarmAsyncAPI:
|
|
|
365
393
|
:param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
|
|
366
394
|
:return: Generator of ArtifactInstance resources
|
|
367
395
|
"""
|
|
396
|
+
ips, urls, domains = self._refang_all(ips), self._refang_all(urls), self._refang_all(domains)
|
|
368
397
|
logger.info('Searching for metadata %s', query)
|
|
369
398
|
async for item in self._paginate(resources.Metadata.get(self, query=query, community=self.community, include=include, exclude=exclude, ips=ips, urls=urls, domains=domains)):
|
|
370
399
|
yield item
|
|
@@ -381,18 +410,28 @@ class PolySwarmAsyncAPI:
|
|
|
381
410
|
async for item in self._paginate(resources.IOC.iocs_by_hash(self, hash_value, hash_type, hide_known_good=hide_known_good, beta=beta)):
|
|
382
411
|
yield item
|
|
383
412
|
|
|
384
|
-
async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None):
|
|
413
|
+
async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
|
|
385
414
|
"""
|
|
386
415
|
Search artifacts by IOC (ip, domain, ttp, or imphash)
|
|
387
|
-
|
|
416
|
+
|
|
388
417
|
:param ip: ip address to search by
|
|
389
418
|
:param domain: domain address to search by
|
|
390
419
|
:param ttp: ttp to search by
|
|
391
420
|
:param imphash: ImpHash to search by
|
|
392
|
-
:
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
421
|
+
:param with_artifacts: True yields a Metadata resource per matching artifact
|
|
422
|
+
(a metadata-search row trimmed to a field set the server owns: artifact.*,
|
|
423
|
+
the scan summary, ssdeep/tlsh and the malware family) instead of its bare
|
|
424
|
+
sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
|
|
425
|
+
(a typed exception); without it a bare call behaves as it always has.
|
|
426
|
+
Needs a server that supports the parameter:
|
|
427
|
+
an older one ignores it and answers bare sha256 strings, which fail to
|
|
428
|
+
parse as Metadata (TypeError).
|
|
429
|
+
:return: Generator of IOC resources whose ``json`` is a sha256 string, or of
|
|
430
|
+
Metadata resources when ``with_artifacts`` is True
|
|
431
|
+
"""
|
|
432
|
+
ip, domain = self._refang(ip), self._refang(domain)
|
|
433
|
+
logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts))
|
|
434
|
+
async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts)):
|
|
396
435
|
yield item
|
|
397
436
|
|
|
398
437
|
async def check_known_hosts(self, ips=[], domains=[]):
|
|
@@ -403,6 +442,7 @@ class PolySwarmAsyncAPI:
|
|
|
403
442
|
:param domains
|
|
404
443
|
:return: Generator of IOC resources
|
|
405
444
|
"""
|
|
445
|
+
ips, domains = self._refang_all(ips), self._refang_all(domains)
|
|
406
446
|
logger.info('Checking known hosts ips: %s, domains: %s', ips, domains)
|
|
407
447
|
async for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
|
|
408
448
|
yield item
|
|
@@ -416,6 +456,7 @@ class PolySwarmAsyncAPI:
|
|
|
416
456
|
:param host
|
|
417
457
|
:return: IOC resource
|
|
418
458
|
"""
|
|
459
|
+
host = self._refang(host)
|
|
419
460
|
logger.info('Creating known good ioc %s %s %s', type, host, source)
|
|
420
461
|
return await self._single(resources.IOC.create_known_good(self, type, host, source))
|
|
421
462
|
|
|
@@ -428,6 +469,7 @@ class PolySwarmAsyncAPI:
|
|
|
428
469
|
:param host
|
|
429
470
|
:return: IOC resource
|
|
430
471
|
"""
|
|
472
|
+
host = self._refang(host)
|
|
431
473
|
logger.info('Creating known bad ioc %s %s %s', type, host, source)
|
|
432
474
|
return await self._single(resources.IOC.create_known_bad(self, type, host, source))
|
|
433
475
|
|
|
@@ -440,6 +482,7 @@ class PolySwarmAsyncAPI:
|
|
|
440
482
|
:param host
|
|
441
483
|
:return: IOC resource
|
|
442
484
|
"""
|
|
485
|
+
host = self._refang(host)
|
|
443
486
|
logger.info('Updating known good ioc %s %s %s %s', id, type, host, source)
|
|
444
487
|
return await self._single(resources.IOC.update_known_good(self, id, type, host, source, good))
|
|
445
488
|
|
|
@@ -1373,11 +1416,12 @@ class PolySwarmAsyncAPI:
|
|
|
1373
1416
|
self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
|
|
1374
1417
|
)
|
|
1375
1418
|
elif artifact_type == resources.ArtifactType.URL:
|
|
1376
|
-
if preprocessing and preprocessing
|
|
1419
|
+
if preprocessing and preprocessing.get("type") == "qrcode":
|
|
1377
1420
|
artifact = resources.LocalArtifact.from_path(
|
|
1378
1421
|
self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
|
|
1379
1422
|
)
|
|
1380
1423
|
else:
|
|
1424
|
+
artifact = self._refang(artifact)
|
|
1381
1425
|
artifact = resources.LocalArtifact.from_content(
|
|
1382
1426
|
self, artifact, artifact_name=artifact_name or artifact,
|
|
1383
1427
|
artifact_type=artifact_type,
|
|
@@ -1480,10 +1524,18 @@ class PolySwarmAsyncAPI:
|
|
|
1480
1524
|
self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
|
|
1481
1525
|
)
|
|
1482
1526
|
elif artifact_type == resources.ArtifactType.URL:
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1527
|
+
# A QR-code submission's argument is an image path, not a URL
|
|
1528
|
+
# (same rule as ``submit``): read the image, never refang it.
|
|
1529
|
+
if preprocessing and preprocessing.get("type") == "qrcode":
|
|
1530
|
+
artifact = resources.LocalArtifact.from_path(
|
|
1531
|
+
self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
|
|
1532
|
+
)
|
|
1533
|
+
else:
|
|
1534
|
+
artifact = self._refang(artifact)
|
|
1535
|
+
artifact = resources.LocalArtifact.from_content(
|
|
1536
|
+
self, artifact, artifact_name=artifact_name or artifact,
|
|
1537
|
+
artifact_type=artifact_type,
|
|
1538
|
+
)
|
|
1487
1539
|
|
|
1488
1540
|
json_params = {
|
|
1489
1541
|
"artifact_name": artifact.artifact_name,
|
|
@@ -1542,6 +1594,7 @@ class PolySwarmAsyncAPI:
|
|
|
1542
1594
|
else:
|
|
1543
1595
|
local = artifact
|
|
1544
1596
|
else:
|
|
1597
|
+
url = self._refang(url)
|
|
1545
1598
|
local = resources.LocalArtifact.from_content(
|
|
1546
1599
|
self, url, artifact_name=artifact_name or url,
|
|
1547
1600
|
artifact_type=resources.ArtifactType.URL,
|
|
@@ -19,7 +19,7 @@ import time
|
|
|
19
19
|
import io
|
|
20
20
|
import logging
|
|
21
21
|
|
|
22
|
-
from polyswarm_api import exceptions, resources, settings
|
|
22
|
+
from polyswarm_api import exceptions, refang, resources, settings
|
|
23
23
|
from polyswarm_api.core import PolyswarmRequest, _as_result_bound
|
|
24
24
|
|
|
25
25
|
from .session import PolyswarmSession
|
|
@@ -55,6 +55,7 @@ class PolyswarmAPI:
|
|
|
55
55
|
verify: bool = True,
|
|
56
56
|
*,
|
|
57
57
|
session: PolyswarmSession | None = None,
|
|
58
|
+
refang_iocs: bool = False,
|
|
58
59
|
**httpx_kwargs,
|
|
59
60
|
):
|
|
60
61
|
key_masked = "******" + (key[-4:] if key and len(key) > 16 else "")
|
|
@@ -69,6 +70,11 @@ class PolyswarmAPI:
|
|
|
69
70
|
self.community = community or settings.DEFAULT_COMMUNITY
|
|
70
71
|
self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
|
|
71
72
|
self.verify = verify
|
|
73
|
+
# Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
|
|
74
|
+
# before building a request. Opt-in: the default preserves the 4.5.0
|
|
75
|
+
# behaviour of sending every input verbatim. See ``polyswarm_api.refang``
|
|
76
|
+
# and ``_refang`` below for exactly which inputs are touched.
|
|
77
|
+
self.refang_iocs = refang_iocs
|
|
72
78
|
self._engines = None
|
|
73
79
|
# Either accept a pre-built session (customization point) or
|
|
74
80
|
# build the default from ``key``. Passing both is ambiguous.
|
|
@@ -95,6 +101,27 @@ class PolyswarmAPI:
|
|
|
95
101
|
)
|
|
96
102
|
return f"<{clsname}({attrs}) at 0x{id(self):x}>"
|
|
97
103
|
|
|
104
|
+
def _refang(self, value):
|
|
105
|
+
"""Refang one URL / domain / IP input when ``refang_iocs`` is on.
|
|
106
|
+
|
|
107
|
+
Applied only to arguments that name a network indicator (a URL to
|
|
108
|
+
search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
|
|
109
|
+
``ip`` / ``domain``, a known-host ``host``) — never to a free-form
|
|
110
|
+
metadata query or a hash.
|
|
111
|
+
Anything that is not a defanged network IoC is returned unchanged.
|
|
112
|
+
"""
|
|
113
|
+
if not self.refang_iocs:
|
|
114
|
+
return value
|
|
115
|
+
return refang.refang_ioc(value)
|
|
116
|
+
|
|
117
|
+
def _refang_all(self, values):
|
|
118
|
+
"""``_refang`` over a list argument; ``None`` / empty pass through."""
|
|
119
|
+
if not values or not self.refang_iocs:
|
|
120
|
+
return values
|
|
121
|
+
if isinstance(values, str):
|
|
122
|
+
return self._refang(values)
|
|
123
|
+
return [self._refang(v) for v in values]
|
|
124
|
+
|
|
98
125
|
def close(self):
|
|
99
126
|
"""Close the underlying HTTP client."""
|
|
100
127
|
self.session.close()
|
|
@@ -366,6 +393,7 @@ class PolyswarmAPI:
|
|
|
366
393
|
:param url: A url to be searched by exact match
|
|
367
394
|
:return: Generator of ArtifactInstance resources
|
|
368
395
|
"""
|
|
396
|
+
url = self._refang(url)
|
|
369
397
|
logger.info("Searching for url %s", url)
|
|
370
398
|
for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
|
|
371
399
|
yield item
|
|
@@ -395,6 +423,11 @@ class PolyswarmAPI:
|
|
|
395
423
|
:param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
|
|
396
424
|
:return: Generator of ArtifactInstance resources
|
|
397
425
|
"""
|
|
426
|
+
ips, urls, domains = (
|
|
427
|
+
self._refang_all(ips),
|
|
428
|
+
self._refang_all(urls),
|
|
429
|
+
self._refang_all(domains),
|
|
430
|
+
)
|
|
398
431
|
logger.info("Searching for metadata %s", query)
|
|
399
432
|
for item in self._paginate(
|
|
400
433
|
resources.Metadata.get(
|
|
@@ -426,7 +459,9 @@ class PolyswarmAPI:
|
|
|
426
459
|
):
|
|
427
460
|
yield item
|
|
428
461
|
|
|
429
|
-
def search_by_ioc(
|
|
462
|
+
def search_by_ioc(
|
|
463
|
+
self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False
|
|
464
|
+
):
|
|
430
465
|
"""
|
|
431
466
|
Search artifacts by IOC (ip, domain, ttp, or imphash)
|
|
432
467
|
|
|
@@ -434,14 +469,36 @@ class PolyswarmAPI:
|
|
|
434
469
|
:param domain: domain address to search by
|
|
435
470
|
:param ttp: ttp to search by
|
|
436
471
|
:param imphash: ImpHash to search by
|
|
437
|
-
:
|
|
438
|
-
|
|
472
|
+
:param with_artifacts: True yields a Metadata resource per matching artifact
|
|
473
|
+
(a metadata-search row trimmed to a field set the server owns: artifact.*,
|
|
474
|
+
the scan summary, ssdeep/tlsh and the malware family) instead of its bare
|
|
475
|
+
sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
|
|
476
|
+
(a typed exception); without it a bare call behaves as it always has.
|
|
477
|
+
Needs a server that supports the parameter:
|
|
478
|
+
an older one ignores it and answers bare sha256 strings, which fail to
|
|
479
|
+
parse as Metadata (TypeError).
|
|
480
|
+
:return: Generator of IOC resources whose ``json`` is a sha256 string, or of
|
|
481
|
+
Metadata resources when ``with_artifacts`` is True
|
|
482
|
+
"""
|
|
483
|
+
ip, domain = self._refang(ip), self._refang(domain)
|
|
439
484
|
logger.info(
|
|
440
|
-
"Searching by ioc %s",
|
|
485
|
+
"Searching by ioc %s",
|
|
486
|
+
dict(
|
|
487
|
+
ip=ip,
|
|
488
|
+
domain=domain,
|
|
489
|
+
ttp=ttp,
|
|
490
|
+
imphash=imphash,
|
|
491
|
+
with_artifacts=with_artifacts,
|
|
492
|
+
),
|
|
441
493
|
)
|
|
442
494
|
for item in self._paginate(
|
|
443
495
|
resources.IOC.ioc_search(
|
|
444
|
-
self,
|
|
496
|
+
self,
|
|
497
|
+
ip=ip,
|
|
498
|
+
domain=domain,
|
|
499
|
+
ttp=ttp,
|
|
500
|
+
imphash=imphash,
|
|
501
|
+
with_artifacts=with_artifacts,
|
|
445
502
|
)
|
|
446
503
|
):
|
|
447
504
|
yield item
|
|
@@ -454,6 +511,7 @@ class PolyswarmAPI:
|
|
|
454
511
|
:param domains
|
|
455
512
|
:return: Generator of IOC resources
|
|
456
513
|
"""
|
|
514
|
+
ips, domains = self._refang_all(ips), self._refang_all(domains)
|
|
457
515
|
logger.info("Checking known hosts ips: %s, domains: %s", ips, domains)
|
|
458
516
|
for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
|
|
459
517
|
yield item
|
|
@@ -467,6 +525,7 @@ class PolyswarmAPI:
|
|
|
467
525
|
:param host
|
|
468
526
|
:return: IOC resource
|
|
469
527
|
"""
|
|
528
|
+
host = self._refang(host)
|
|
470
529
|
logger.info("Creating known good ioc %s %s %s", type, host, source)
|
|
471
530
|
return self._single(resources.IOC.create_known_good(self, type, host, source))
|
|
472
531
|
|
|
@@ -479,6 +538,7 @@ class PolyswarmAPI:
|
|
|
479
538
|
:param host
|
|
480
539
|
:return: IOC resource
|
|
481
540
|
"""
|
|
541
|
+
host = self._refang(host)
|
|
482
542
|
logger.info("Creating known bad ioc %s %s %s", type, host, source)
|
|
483
543
|
return self._single(resources.IOC.create_known_bad(self, type, host, source))
|
|
484
544
|
|
|
@@ -491,6 +551,7 @@ class PolyswarmAPI:
|
|
|
491
551
|
:param host
|
|
492
552
|
:return: IOC resource
|
|
493
553
|
"""
|
|
554
|
+
host = self._refang(host)
|
|
494
555
|
logger.info("Updating known good ioc %s %s %s %s", id, type, host, source)
|
|
495
556
|
return self._single(
|
|
496
557
|
resources.IOC.update_known_good(self, id, type, host, source, good)
|
|
@@ -1700,7 +1761,7 @@ class PolyswarmAPI:
|
|
|
1700
1761
|
artifact_name=artifact_name,
|
|
1701
1762
|
)
|
|
1702
1763
|
elif artifact_type == resources.ArtifactType.URL:
|
|
1703
|
-
if preprocessing and preprocessing
|
|
1764
|
+
if preprocessing and preprocessing.get("type") == "qrcode":
|
|
1704
1765
|
artifact = resources.LocalArtifact.from_path(
|
|
1705
1766
|
self,
|
|
1706
1767
|
artifact,
|
|
@@ -1708,6 +1769,7 @@ class PolyswarmAPI:
|
|
|
1708
1769
|
artifact_name=artifact_name,
|
|
1709
1770
|
)
|
|
1710
1771
|
else:
|
|
1772
|
+
artifact = self._refang(artifact)
|
|
1711
1773
|
artifact = resources.LocalArtifact.from_content(
|
|
1712
1774
|
self,
|
|
1713
1775
|
artifact,
|
|
@@ -1822,12 +1884,23 @@ class PolyswarmAPI:
|
|
|
1822
1884
|
artifact_name=artifact_name,
|
|
1823
1885
|
)
|
|
1824
1886
|
elif artifact_type == resources.ArtifactType.URL:
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1887
|
+
# A QR-code submission's argument is an image path, not a URL
|
|
1888
|
+
# (same rule as ``submit``): read the image, never refang it.
|
|
1889
|
+
if preprocessing and preprocessing.get("type") == "qrcode":
|
|
1890
|
+
artifact = resources.LocalArtifact.from_path(
|
|
1891
|
+
self,
|
|
1892
|
+
artifact,
|
|
1893
|
+
artifact_type=artifact_type,
|
|
1894
|
+
artifact_name=artifact_name,
|
|
1895
|
+
)
|
|
1896
|
+
else:
|
|
1897
|
+
artifact = self._refang(artifact)
|
|
1898
|
+
artifact = resources.LocalArtifact.from_content(
|
|
1899
|
+
self,
|
|
1900
|
+
artifact,
|
|
1901
|
+
artifact_name=artifact_name or artifact,
|
|
1902
|
+
artifact_type=artifact_type,
|
|
1903
|
+
)
|
|
1831
1904
|
|
|
1832
1905
|
json_params = {
|
|
1833
1906
|
"artifact_name": artifact.artifact_name,
|
|
@@ -1896,6 +1969,7 @@ class PolyswarmAPI:
|
|
|
1896
1969
|
else:
|
|
1897
1970
|
local = artifact
|
|
1898
1971
|
else:
|
|
1972
|
+
url = self._refang(url)
|
|
1899
1973
|
local = resources.LocalArtifact.from_content(
|
|
1900
1974
|
self,
|
|
1901
1975
|
url,
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Refang defanged indicators of compromise (IoCs).
|
|
2
|
+
|
|
3
|
+
Portability is part of the contract, because the same pattern can match
|
|
4
|
+
different characters in different regex engines. So: no case-insensitive
|
|
5
|
+
flag (Python's folds Unicode, e.g. U+212A KELVIN SIGN matches ``k``) — letters
|
|
6
|
+
are spelled as explicit ``[aA]`` classes; no ``\\b``, ``\\d``, ``\\w``, ``\\s``
|
|
7
|
+
or ``\\S`` (their Unicode sets differ between engines) — whitespace is the
|
|
8
|
+
explicit ASCII set ``[ \\t\\n\\r\\f\\v]``, and trimming strips only that set;
|
|
9
|
+
and full matches use ``fullmatch`` (Python's ``$`` also matches before a
|
|
10
|
+
trailing newline).
|
|
11
|
+
|
|
12
|
+
What the contract is, why clients refang at their own edge, where the client
|
|
13
|
+
applies it and what is out of scope: ``specs/05-downstream-contract.md``
|
|
14
|
+
§"IoC refanging".
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
__all__ = ['is_network_ioc', 'refang_ioc', 'refang_text']
|
|
20
|
+
|
|
21
|
+
# ASCII whitespace only — see the module docstring on portability.
|
|
22
|
+
_WS_CHARS = ' \t\n\r\f\v'
|
|
23
|
+
_WS = r'[ \t\n\r\f\v]'
|
|
24
|
+
|
|
25
|
+
# Applied in order. The bracket rules run first so that ``hxxps[:]//`` has
|
|
26
|
+
# become ``hxxps://`` by the time the (anchored) scheme rules look for ``://``.
|
|
27
|
+
# ``hxxps`` is matched before ``hxxp``: the shorter rule would otherwise leave
|
|
28
|
+
# a stray ``s`` behind. Each scheme rule is a literal replacement — a capture
|
|
29
|
+
# group would carry the input's case (``HXXPS`` -> ``httpS``).
|
|
30
|
+
_RULES = (
|
|
31
|
+
(re.compile(rf'[\[({{]{_WS}*:{_WS}*/{_WS}*/{_WS}*[\])}}]'), '://'),
|
|
32
|
+
(re.compile(rf'[\[({{]{_WS}*:{_WS}*[\])}}]'), ':'),
|
|
33
|
+
(re.compile(rf'[\[({{]{_WS}*/{_WS}*[\])}}]'), '/'),
|
|
34
|
+
(re.compile(rf'[\[({{]{_WS}*\.{_WS}*[\])}}]'), '.'),
|
|
35
|
+
(re.compile(rf'[\[({{]{_WS}*[dD][oO][tT]{_WS}*[\])}}]'), '.'),
|
|
36
|
+
(re.compile(r'^[hH][xX*]{2}[pP][sS](?=://)'), 'https'),
|
|
37
|
+
(re.compile(r'^[hH][xX*]{2}[pP](?=://)'), 'http'),
|
|
38
|
+
(re.compile(r'^[fF][xX][pP][sS](?=://)'), 'ftps'),
|
|
39
|
+
(re.compile(r'^[fF][xX][pP](?=://)'), 'ftp'),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
_OCTET = r'(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
|
|
43
|
+
_IPV4 = rf'{_OCTET}(?:\.{_OCTET}){{3}}'
|
|
44
|
+
_LABEL = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?'
|
|
45
|
+
_TLD = r'(?:[a-zA-Z]{2,63}|[xX][nN]--[a-zA-Z0-9-]{1,59})'
|
|
46
|
+
_DOMAIN = rf'(?:{_LABEL}\.)+{_TLD}'
|
|
47
|
+
_IPV6 = r'\[[0-9a-fA-F:.]+\]'
|
|
48
|
+
_HOST = rf'(?:{_DOMAIN}|{_IPV4}|{_IPV6})'
|
|
49
|
+
_SCHEME = r'(?:[hH][tT][tT][pP][sS]?|[fF][tT][pP][sS]?)'
|
|
50
|
+
# Used with ``fullmatch`` — never ``^…$``, whose ``$`` accepts a trailing ``\n``.
|
|
51
|
+
_NETWORK_IOC = re.compile(
|
|
52
|
+
rf'(?:{_SCHEME}://)?(?:[^ \t\n\r\f\v/?#@]+@)?{_HOST}(?::[0-9]{{1,5}})?(?:[/?#][^ \t\n\r\f\v]*)?'
|
|
53
|
+
)
|
|
54
|
+
# An optional scheme-like token (live or defanged, e.g. ``hxxps``) plus the
|
|
55
|
+
# host: everything before the first ``/``, ``?`` or ``#``.
|
|
56
|
+
_SCHEME_AND_HOST = re.compile(r'^(?:[a-zA-Z*]+://)?[^/?#]*')
|
|
57
|
+
_QUERY_SYNTAX = re.compile(r'[ \t\n\r\f\v"]')
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def refang_text(text):
|
|
61
|
+
"""Apply every refang rewrite to ``text``, with no trimming and no gate.
|
|
62
|
+
|
|
63
|
+
Ungated: it rewrites anything that contains a defang token, IoC or not.
|
|
64
|
+
Callers that take user input want :func:`refang_ioc` instead.
|
|
65
|
+
"""
|
|
66
|
+
for pattern, replacement in _RULES:
|
|
67
|
+
text = pattern.sub(replacement, text)
|
|
68
|
+
return text
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def is_network_ioc(candidate):
|
|
72
|
+
"""Whether ``candidate`` is a URL, a domain, or an IP address.
|
|
73
|
+
|
|
74
|
+
A URL may carry an http(s)/ftp(s) scheme, userinfo, a port and a path; an
|
|
75
|
+
IPv6 host is accepted only in its bracketed URL form.
|
|
76
|
+
"""
|
|
77
|
+
return bool(_NETWORK_IOC.fullmatch(candidate))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def refang_ioc(value, accept=None):
|
|
81
|
+
"""Return the refanged form of ``value`` if it is a defanged network IoC.
|
|
82
|
+
|
|
83
|
+
Anything else comes back exactly as given (untrimmed), so this is safe to
|
|
84
|
+
run on every IoC-shaped input. The gate, in order:
|
|
85
|
+
|
|
86
|
+
* nothing was defanged -> unchanged;
|
|
87
|
+
* the rewrite contains ASCII whitespace or a double quote -> unchanged: that is
|
|
88
|
+
a query or quoted data, never a single indicator;
|
|
89
|
+
* the rewrite keeps the input's scheme and host intact (with or without
|
|
90
|
+
a scheme) -> unchanged: only a path, query or fragment would change,
|
|
91
|
+
so a legitimate ``[.]`` in ``example.com/a[.]b`` survives;
|
|
92
|
+
* the rewrite is not a URL, domain or IP -> unchanged;
|
|
93
|
+
* ``accept`` (optional) rejects the rewrite -> unchanged. Callers use it
|
|
94
|
+
to require their own routing to agree, e.g. "this would be searched as
|
|
95
|
+
a URL".
|
|
96
|
+
|
|
97
|
+
A refanged value is returned trimmed of ASCII whitespace.
|
|
98
|
+
"""
|
|
99
|
+
if not isinstance(value, str):
|
|
100
|
+
return value
|
|
101
|
+
trimmed = value.strip(_WS_CHARS)
|
|
102
|
+
candidate = refang_text(trimmed)
|
|
103
|
+
if candidate == trimmed:
|
|
104
|
+
return value
|
|
105
|
+
if _QUERY_SYNTAX.search(candidate):
|
|
106
|
+
return value
|
|
107
|
+
if candidate.startswith(_SCHEME_AND_HOST.match(trimmed).group(0)):
|
|
108
|
+
return value
|
|
109
|
+
if not is_network_ioc(candidate):
|
|
110
|
+
return value
|
|
111
|
+
if accept is not None and not accept(candidate):
|
|
112
|
+
return value
|
|
113
|
+
return candidate
|
|
@@ -169,7 +169,7 @@ class IOC(core.BaseJsonResource):
|
|
|
169
169
|
)
|
|
170
170
|
|
|
171
171
|
@classmethod
|
|
172
|
-
def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None):
|
|
172
|
+
def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
|
|
173
173
|
params = dict(community=api.community)
|
|
174
174
|
if ip is not None:
|
|
175
175
|
params['ip'] = ip
|
|
@@ -179,12 +179,17 @@ class IOC(core.BaseJsonResource):
|
|
|
179
179
|
params['ttp'] = ttp
|
|
180
180
|
if imphash is not None:
|
|
181
181
|
params['imphash'] = imphash
|
|
182
|
+
if with_artifacts:
|
|
183
|
+
# An int, not a bool: the session renders bools as 'True', which the
|
|
184
|
+
# server's boolean parser refuses (it accepts only 0/1/false/true).
|
|
185
|
+
params['with_artifacts'] = 1
|
|
182
186
|
return core.PolyswarmRequest(
|
|
183
187
|
api=api,
|
|
184
188
|
method='GET',
|
|
185
189
|
url=f'{api.uri}/ioc/search',
|
|
186
190
|
params=params,
|
|
187
|
-
|
|
191
|
+
# Opt-in rows are metadata-search documents; the default is bare sha256s.
|
|
192
|
+
result_parser=Metadata if with_artifacts else cls,
|
|
188
193
|
)
|
|
189
194
|
|
|
190
195
|
@classmethod
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|