polyswarm-api 4.1.0__tar.gz → 4.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {polyswarm_api-4.1.0/src/polyswarm_api.egg-info → polyswarm_api-4.3.0}/PKG-INFO +1 -1
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/pyproject.toml +14 -9
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/__init__.py +1 -1
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/aio/api.py +68 -8
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/aio/session.py +13 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/api.py +67 -7
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/core.py +31 -2
- polyswarm_api-4.3.0/src/polyswarm_api/exceptions.py +135 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/resources.py +8 -3
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/session.py +14 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0/src/polyswarm_api.egg-info}/PKG-INFO +1 -1
- polyswarm_api-4.1.0/src/polyswarm_api/exceptions.py +0 -85
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/LICENSE +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/README.md +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/setup.cfg +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/aio/__init__.py +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api/settings.py +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api.egg-info/SOURCES.txt +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api.egg-info/dependency_links.txt +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api.egg-info/requires.txt +0 -0
- {polyswarm_api-4.1.0 → polyswarm_api-4.3.0}/src/polyswarm_api.egg-info/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "polyswarm_api"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.3.0"
|
|
8
8
|
description = "Client library to simplify interacting with the PolySwarm consumer API"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10,<4"
|
|
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
|
|
|
55
55
|
where = ["src"]
|
|
56
56
|
|
|
57
57
|
[tool.bumpversion]
|
|
58
|
-
current_version = "4.
|
|
58
|
+
current_version = "4.3.0"
|
|
59
59
|
commit = true
|
|
60
60
|
tag = false
|
|
61
61
|
sign_tags = true
|
|
@@ -76,11 +76,17 @@ search = "__version__ = '{current_version}'"
|
|
|
76
76
|
replace = "__version__ = '{new_version}'"
|
|
77
77
|
|
|
78
78
|
[tool.pytest.ini_options]
|
|
79
|
-
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
#
|
|
79
|
+
# Live logging OFF by default. When on, pytest streams log records as the run goes
|
|
80
|
+
# AND forces every test onto its own nodeid line (verbose-style) — ~130 lines of
|
|
81
|
+
# pure noise on a green xdist run. With it off the run prints dots + an end
|
|
82
|
+
# summary, and a FAILING test still shows its captured logs (at log_level, below)
|
|
83
|
+
# plus the traceback. Opt back into live streaming with TESTS_LOG_CLI=1 (or an
|
|
84
|
+
# explicit --log-cli-level) when debugging a live run; see test/conftest.py.
|
|
85
|
+
log_cli = false
|
|
86
|
+
# Level used for live logging when it's opted into (TESTS_LOG_CLI=1). INFO keeps
|
|
87
|
+
# the useful app request/response lines while dropping the DEBUG firehose (VCR
|
|
88
|
+
# per-request match trace, parse traces) that otherwise bloats logs ~1000x.
|
|
89
|
+
# TESTS_LOG_LEVEL overrides both this and the captured-log level (conftest.py).
|
|
84
90
|
log_cli_level = "INFO"
|
|
85
91
|
log_format = "%(asctime)s %(levelname)-2s [%(name)s:%(filename)s:%(lineno)d:%(funcName)1s()] %(message)s"
|
|
86
92
|
python_files = ["test_*.py", "*_test.py"]
|
|
@@ -95,7 +101,6 @@ filterwarnings = [
|
|
|
95
101
|
asyncio_mode = "auto"
|
|
96
102
|
addopts = [
|
|
97
103
|
"--continue-on-collection-errors",
|
|
98
|
-
"-v",
|
|
99
104
|
"-s",
|
|
100
|
-
"-
|
|
105
|
+
"-ra",
|
|
101
106
|
]
|
|
@@ -939,6 +939,13 @@ class PolySwarmAsyncAPI:
|
|
|
939
939
|
:param fh: A file-like object which we are going to write the contents of the artifact to.
|
|
940
940
|
:param hash_type: Hash type of the provided hash_. Will attempt to auto-detect if not explicitly provided.
|
|
941
941
|
:return: A LocalHandle resource
|
|
942
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
943
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
944
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
945
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
946
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
947
|
+
the status is checked before any write (the handle is the caller's and arrives
|
|
948
|
+
already open, so "before the file is opened" would be the wrong reason here).
|
|
942
949
|
"""
|
|
943
950
|
logger.info('Downloading %s into handle', hash_)
|
|
944
951
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -1025,7 +1032,8 @@ class PolySwarmAsyncAPI:
|
|
|
1025
1032
|
report_task = await self._single(resources.ReportLLMPostProcessing.create(self,
|
|
1026
1033
|
instance_id=instance_id,
|
|
1027
1034
|
cape_sandbox_task_id=cape_sandbox_task_id,
|
|
1028
|
-
triage_sandbox_task_id=triage_sandbox_task_id
|
|
1035
|
+
triage_sandbox_task_id=triage_sandbox_task_id,
|
|
1036
|
+
community=self.community))
|
|
1029
1037
|
return report_task
|
|
1030
1038
|
|
|
1031
1039
|
async def llm_report_get(self, report_task_id):
|
|
@@ -1598,6 +1606,12 @@ class PolySwarmAsyncAPI:
|
|
|
1598
1606
|
:param hash_: Hashable (Artifact, LocalArtifact, Hash) or hex-encoded SHA256/SHA1/MD5.
|
|
1599
1607
|
:param hash_type: Hash type; auto-detected from the literal if not provided.
|
|
1600
1608
|
:return: A ``LocalArtifact`` resource with its handle already closed.
|
|
1609
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1610
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1611
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1612
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1613
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1614
|
+
the status is checked before the file is opened.
|
|
1601
1615
|
"""
|
|
1602
1616
|
logger.info('Downloading %s into %s', hash_, out_dir)
|
|
1603
1617
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -1608,7 +1622,15 @@ class PolySwarmAsyncAPI:
|
|
|
1608
1622
|
return artifact
|
|
1609
1623
|
|
|
1610
1624
|
async def download_id(self, out_dir, instance_id):
|
|
1611
|
-
"""Download an artifact by its instance id into ``out_dir``.
|
|
1625
|
+
"""Download an artifact by its instance id into ``out_dir``.
|
|
1626
|
+
|
|
1627
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1628
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1629
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1630
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1631
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1632
|
+
the status is checked before the file is opened.
|
|
1633
|
+
"""
|
|
1612
1634
|
logger.info('Downloading %s into %s', instance_id, out_dir)
|
|
1613
1635
|
artifact = await self._single(
|
|
1614
1636
|
resources.LocalArtifact.download_id(self, instance_id, folder=out_dir),
|
|
@@ -1617,7 +1639,21 @@ class PolySwarmAsyncAPI:
|
|
|
1617
1639
|
return artifact
|
|
1618
1640
|
|
|
1619
1641
|
async def download_sandbox_artifact(self, out_dir, sandbox_task_id, instance_id):
|
|
1620
|
-
"""Download a sandbox-produced artifact (e.g. PCAP, dropped file) into ``out_dir``.
|
|
1642
|
+
"""Download a sandbox-produced artifact (e.g. PCAP, dropped file) into ``out_dir``.
|
|
1643
|
+
|
|
1644
|
+
The known-good gate applies to **every** sandbox artifact by its own sha256 —
|
|
1645
|
+
dropped file, screenshot, report, … — with no sample-vs-evidence carve-out
|
|
1646
|
+
server-side, so any member whose digest is catalogued (and eligible) raises below.
|
|
1647
|
+
In practice that is almost always a dropped file; evidence collides only on byte
|
|
1648
|
+
identity, and the empty-file digest is refused at the catalogue boundary instead.
|
|
1649
|
+
|
|
1650
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1651
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1652
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1653
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1654
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1655
|
+
the status is checked before the file is opened.
|
|
1656
|
+
"""
|
|
1621
1657
|
logger.info('Downloading sandbox artifact %s %s', sandbox_task_id, instance_id)
|
|
1622
1658
|
sandbox_artifact = await self._single(
|
|
1623
1659
|
resources.LocalArtifact.download_sandbox_artifact(
|
|
@@ -1641,8 +1677,29 @@ class PolySwarmAsyncAPI:
|
|
|
1641
1677
|
|
|
1642
1678
|
:param hash_: Hashable (Artifact, LocalArtifact, Hash) or hex-encoded SHA256/SHA1/MD5.
|
|
1643
1679
|
:param hash_type: Hash type; auto-detected if not provided.
|
|
1644
|
-
:param require_scan: If True, only count artifacts that have been scanned
|
|
1680
|
+
:param require_scan: If True, only count artifacts that have been *scanned*.
|
|
1681
|
+
Being catalogued as known-good is **not** a scan: the platform never scans
|
|
1682
|
+
such a sample, so a hash whose only record is a known-good reference reports
|
|
1683
|
+
absent under ``require_scan`` (and present without it, because that reference
|
|
1684
|
+
is a real searchable record).
|
|
1645
1685
|
:return: ``True`` if the artifact exists in PolySwarm's index.
|
|
1686
|
+
|
|
1687
|
+
.. note::
|
|
1688
|
+
This is an existence probe, and its status codes are a **frozen contract**:
|
|
1689
|
+
``200`` means found, ``204`` means not found, and anything else means the
|
|
1690
|
+
*request* was wrong.
|
|
1691
|
+
|
|
1692
|
+
Only ``200`` is ``True``. ``204`` and ``404`` are both ``False`` — the server
|
|
1693
|
+
answers ``204`` for absent, and ``404`` is tolerated as absent for historical
|
|
1694
|
+
reasons.
|
|
1695
|
+
|
|
1696
|
+
A non-2xx status does not raise here, and the reason is the **method**, not a
|
|
1697
|
+
missing result parser: ``parse_response`` short-circuits ``HEAD``, setting the
|
|
1698
|
+
status code as the result and returning before it reaches the non-2xx→exception
|
|
1699
|
+
mapping (which otherwise applies whether or not a parser is set). So on this
|
|
1700
|
+
endpoint a server error also collapses to ``False``, indistinguishable from a
|
|
1701
|
+
genuine "does not exist" — which is why neither side may widen what counts as
|
|
1702
|
+
found. See ``specs/03-endpoints.md``.
|
|
1646
1703
|
"""
|
|
1647
1704
|
logger.info('Exists for hash %s', hash_)
|
|
1648
1705
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -1651,8 +1708,11 @@ class PolySwarmAsyncAPI:
|
|
|
1651
1708
|
self, hash_.hash, hash_.hash_type, require_scan=require_scan,
|
|
1652
1709
|
),
|
|
1653
1710
|
)
|
|
1654
|
-
# exists_hash is a HEAD; ``result`` is the status code.
|
|
1655
|
-
#
|
|
1656
|
-
#
|
|
1657
|
-
|
|
1711
|
+
# exists_hash is a HEAD; ``result`` is the status code. The endpoint returns
|
|
1712
|
+
# 200 when the artifact is present and 204 when it is absent ("the request
|
|
1713
|
+
# worked, but there was no matching artifact, so nothing was returned"). Only
|
|
1714
|
+
# a 200 means the artifact exists — 204 (and 404) both mean "absent" → False.
|
|
1715
|
+
# NB: this is deliberately ``== 200``, not a ``// 100 == 2`` 2xx check, because
|
|
1716
|
+
# 204 is a successful 2xx that means the opposite of "exists".
|
|
1717
|
+
return int(result) == 200
|
|
1658
1718
|
|
|
@@ -148,6 +148,19 @@ class AsyncPolyswarmSession:
|
|
|
148
148
|
await response.aread()
|
|
149
149
|
_raise_for_status(response, request)
|
|
150
150
|
|
|
151
|
+
if response.status_code == 204:
|
|
152
|
+
# A 204 is the "no data / absent" signal on a download: the request
|
|
153
|
+
# succeeded but there is nothing to write ("the request worked, but
|
|
154
|
+
# there was no matching artifact, so nothing was returned"). The
|
|
155
|
+
# shared ``parse_response`` maps a parser-backed 204 to
|
|
156
|
+
# ``NoResultsException``; the streaming path bypasses ``parse_response``,
|
|
157
|
+
# so mirror that rule explicitly here. Without it, an absent artifact
|
|
158
|
+
# would be written out as a successful *empty* file instead of
|
|
159
|
+
# surfacing "no results".
|
|
160
|
+
raise exceptions.NoResultsException(
|
|
161
|
+
request, 'The request returned no results.',
|
|
162
|
+
)
|
|
163
|
+
|
|
151
164
|
pk = request.parser_kwargs or {}
|
|
152
165
|
handle, name, created = request.result_parser.open_destination(
|
|
153
166
|
pk.get('folder'), pk.get('handle'), pk.get('artifact_name'), response,
|
|
@@ -1139,6 +1139,13 @@ class PolyswarmAPI:
|
|
|
1139
1139
|
:param fh: A file-like object which we are going to write the contents of the artifact to.
|
|
1140
1140
|
:param hash_type: Hash type of the provided hash_. Will attempt to auto-detect if not explicitly provided.
|
|
1141
1141
|
:return: A LocalHandle resource
|
|
1142
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1143
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1144
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1145
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1146
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1147
|
+
the status is checked before any write (the handle is the caller's and arrives
|
|
1148
|
+
already open, so "before the file is opened" would be the wrong reason here).
|
|
1142
1149
|
"""
|
|
1143
1150
|
logger.info("Downloading %s into handle", hash_)
|
|
1144
1151
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -1262,6 +1269,7 @@ class PolyswarmAPI:
|
|
|
1262
1269
|
instance_id=instance_id,
|
|
1263
1270
|
cape_sandbox_task_id=cape_sandbox_task_id,
|
|
1264
1271
|
triage_sandbox_task_id=triage_sandbox_task_id,
|
|
1272
|
+
community=self.community,
|
|
1265
1273
|
)
|
|
1266
1274
|
)
|
|
1267
1275
|
return report_task
|
|
@@ -1940,6 +1948,12 @@ class PolyswarmAPI:
|
|
|
1940
1948
|
:param hash_: Hashable (Artifact, LocalArtifact, Hash) or hex-encoded SHA256/SHA1/MD5.
|
|
1941
1949
|
:param hash_type: Hash type; auto-detected from the literal if not provided.
|
|
1942
1950
|
:return: A ``LocalArtifact`` resource with its handle already closed.
|
|
1951
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1952
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1953
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1954
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1955
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1956
|
+
the status is checked before the file is opened.
|
|
1943
1957
|
"""
|
|
1944
1958
|
logger.info("Downloading %s into %s", hash_, out_dir)
|
|
1945
1959
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -1952,7 +1966,15 @@ class PolyswarmAPI:
|
|
|
1952
1966
|
return artifact
|
|
1953
1967
|
|
|
1954
1968
|
def download_id(self, out_dir, instance_id):
|
|
1955
|
-
"""Download an artifact by its instance id into ``out_dir``.
|
|
1969
|
+
"""Download an artifact by its instance id into ``out_dir``.
|
|
1970
|
+
|
|
1971
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1972
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1973
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1974
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1975
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1976
|
+
the status is checked before the file is opened.
|
|
1977
|
+
"""
|
|
1956
1978
|
logger.info("Downloading %s into %s", instance_id, out_dir)
|
|
1957
1979
|
artifact = self._single(
|
|
1958
1980
|
resources.LocalArtifact.download_id(self, instance_id, folder=out_dir),
|
|
@@ -1961,7 +1983,21 @@ class PolyswarmAPI:
|
|
|
1961
1983
|
return artifact
|
|
1962
1984
|
|
|
1963
1985
|
def download_sandbox_artifact(self, out_dir, sandbox_task_id, instance_id):
|
|
1964
|
-
"""Download a sandbox-produced artifact (e.g. PCAP, dropped file) into ``out_dir``.
|
|
1986
|
+
"""Download a sandbox-produced artifact (e.g. PCAP, dropped file) into ``out_dir``.
|
|
1987
|
+
|
|
1988
|
+
The known-good gate applies to **every** sandbox artifact by its own sha256 —
|
|
1989
|
+
dropped file, screenshot, report, … — with no sample-vs-evidence carve-out
|
|
1990
|
+
server-side, so any member whose digest is catalogued (and eligible) raises below.
|
|
1991
|
+
In practice that is almost always a dropped file; evidence collides only on byte
|
|
1992
|
+
identity, and the empty-file digest is refused at the catalogue boundary instead.
|
|
1993
|
+
|
|
1994
|
+
:raises KnownGoodWithheldException: the sha256 is catalogued as a known-good binary,
|
|
1995
|
+
so its bytes are withheld by design. A ``NotFoundException`` subclass, so existing
|
|
1996
|
+
``except NotFoundException`` handling still catches it; catch it specifically to
|
|
1997
|
+
tell a deliberate refusal apart from a missing artifact, and read
|
|
1998
|
+
``.sources`` for the feeds that flagged it. Nothing is written to the destination —
|
|
1999
|
+
the status is checked before the file is opened.
|
|
2000
|
+
"""
|
|
1965
2001
|
logger.info("Downloading sandbox artifact %s %s", sandbox_task_id, instance_id)
|
|
1966
2002
|
sandbox_artifact = self._single(
|
|
1967
2003
|
resources.LocalArtifact.download_sandbox_artifact(
|
|
@@ -1988,8 +2024,29 @@ class PolyswarmAPI:
|
|
|
1988
2024
|
|
|
1989
2025
|
:param hash_: Hashable (Artifact, LocalArtifact, Hash) or hex-encoded SHA256/SHA1/MD5.
|
|
1990
2026
|
:param hash_type: Hash type; auto-detected if not provided.
|
|
1991
|
-
:param require_scan: If True, only count artifacts that have been scanned
|
|
2027
|
+
:param require_scan: If True, only count artifacts that have been *scanned*.
|
|
2028
|
+
Being catalogued as known-good is **not** a scan: the platform never scans
|
|
2029
|
+
such a sample, so a hash whose only record is a known-good reference reports
|
|
2030
|
+
absent under ``require_scan`` (and present without it, because that reference
|
|
2031
|
+
is a real searchable record).
|
|
1992
2032
|
:return: ``True`` if the artifact exists in PolySwarm's index.
|
|
2033
|
+
|
|
2034
|
+
.. note::
|
|
2035
|
+
This is an existence probe, and its status codes are a **frozen contract**:
|
|
2036
|
+
``200`` means found, ``204`` means not found, and anything else means the
|
|
2037
|
+
*request* was wrong.
|
|
2038
|
+
|
|
2039
|
+
Only ``200`` is ``True``. ``204`` and ``404`` are both ``False`` — the server
|
|
2040
|
+
answers ``204`` for absent, and ``404`` is tolerated as absent for historical
|
|
2041
|
+
reasons.
|
|
2042
|
+
|
|
2043
|
+
A non-2xx status does not raise here, and the reason is the **method**, not a
|
|
2044
|
+
missing result parser: ``parse_response`` short-circuits ``HEAD``, setting the
|
|
2045
|
+
status code as the result and returning before it reaches the non-2xx→exception
|
|
2046
|
+
mapping (which otherwise applies whether or not a parser is set). So on this
|
|
2047
|
+
endpoint a server error also collapses to ``False``, indistinguishable from a
|
|
2048
|
+
genuine "does not exist" — which is why neither side may widen what counts as
|
|
2049
|
+
found. See ``specs/03-endpoints.md``.
|
|
1993
2050
|
"""
|
|
1994
2051
|
logger.info("Exists for hash %s", hash_)
|
|
1995
2052
|
hash_ = resources.Hash.from_hashable(hash_, hash_type=hash_type)
|
|
@@ -2001,7 +2058,10 @@ class PolyswarmAPI:
|
|
|
2001
2058
|
require_scan=require_scan,
|
|
2002
2059
|
),
|
|
2003
2060
|
)
|
|
2004
|
-
# exists_hash is a HEAD; ``result`` is the status code.
|
|
2005
|
-
#
|
|
2006
|
-
#
|
|
2007
|
-
|
|
2061
|
+
# exists_hash is a HEAD; ``result`` is the status code. The endpoint returns
|
|
2062
|
+
# 200 when the artifact is present and 204 when it is absent ("the request
|
|
2063
|
+
# worked, but there was no matching artifact, so nothing was returned"). Only
|
|
2064
|
+
# a 200 means the artifact exists — 204 (and 404) both mean "absent" → False.
|
|
2065
|
+
# NB: this is deliberately ``== 200``, not a ``// 100 == 2`` 2xx check, because
|
|
2066
|
+
# 204 is a successful 2xx that means the opposite of "exists".
|
|
2067
|
+
return int(result) == 200
|
|
@@ -330,7 +330,9 @@ def _raise_for_status(response, request):
|
|
|
330
330
|
|
|
331
331
|
Shared by ``parse_response`` (buffered/JSON path) and the session's
|
|
332
332
|
streaming-download path, so both raise identically (429/404/422/other,
|
|
333
|
-
plus the non-JSON-body fallbacks).
|
|
333
|
+
plus the non-JSON-body fallbacks). A 404 whose ``errors`` payload carries the
|
|
334
|
+
``KNOWN_GOOD`` code raises the ``NotFoundException`` subclass
|
|
335
|
+
``KnownGoodWithheldException``. The response body must already be
|
|
334
336
|
readable — streaming callers ``read()`` / ``aread()`` it first. **Always
|
|
335
337
|
raises; never returns.**
|
|
336
338
|
"""
|
|
@@ -354,6 +356,17 @@ def _raise_for_status(response, request):
|
|
|
354
356
|
)
|
|
355
357
|
raise exceptions.UsageLimitsExceededException(request, message)
|
|
356
358
|
elif request.status_code == 404:
|
|
359
|
+
# A download refused because the artifact is a known-good binary carries a
|
|
360
|
+
# machine-readable code in the error envelope's ``errors`` slot. Raise the
|
|
361
|
+
# ``NotFoundException`` subclass so callers can tell "withheld by design"
|
|
362
|
+
# apart from a plain miss; every other 404 stays a plain NotFoundException.
|
|
363
|
+
# The raw ``sources`` payload goes in as-is — the exception normalises it
|
|
364
|
+
# into the documented list-of-feed-names shape.
|
|
365
|
+
errors = request.errors
|
|
366
|
+
if isinstance(errors, dict) and errors.get('code') == 'KNOWN_GOOD':
|
|
367
|
+
raise exceptions.KnownGoodWithheldException(
|
|
368
|
+
request, request._result, sources=errors.get('sources'),
|
|
369
|
+
)
|
|
357
370
|
raise exceptions.NotFoundException(request, request._result)
|
|
358
371
|
elif request.status_code == 422:
|
|
359
372
|
raise exceptions.FailedInstanceException(request, request._result)
|
|
@@ -383,7 +396,23 @@ def _bad_status_message(request):
|
|
|
383
396
|
f'Message: {request._result}'
|
|
384
397
|
)
|
|
385
398
|
if request.errors:
|
|
386
|
-
errors
|
|
399
|
+
# Two ``errors`` shapes are in play. The legacy shape is a **list** of
|
|
400
|
+
# per-error entries — one rendered line each. The way-forward shape is a
|
|
401
|
+
# **mapping** carrying the machine-readable code plus its context
|
|
402
|
+
# (``{'code': 'KNOWN_GOOD', 'known_good': True, 'sources': [...]}``), and the
|
|
403
|
+
# server forwards it on every status, not just the 404 the code was
|
|
404
|
+
# introduced for. Iterating a mapping yields only its KEYS, so rendering it
|
|
405
|
+
# like a list would drop every value from the message the caller sees —
|
|
406
|
+
# render it as ``key=value`` lines instead.
|
|
407
|
+
# The same reasoning applies to a bare string: iterating it yields characters, so
|
|
408
|
+
# `"errors": "some prose"` would render one letter per line. Only genuinely
|
|
409
|
+
# sequence-shaped payloads get the line-per-entry treatment.
|
|
410
|
+
if isinstance(request.errors, dict):
|
|
411
|
+
errors = '\n'.join(f'{k}={v}' for k, v in request.errors.items())
|
|
412
|
+
elif isinstance(request.errors, (list, tuple)):
|
|
413
|
+
errors = '\n'.join(str(error) for error in request.errors)
|
|
414
|
+
else:
|
|
415
|
+
errors = str(request.errors)
|
|
387
416
|
message = f'{message}\nErrors:\n{errors}'
|
|
388
417
|
return message
|
|
389
418
|
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
logger = logging.getLogger(__name__)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class PolyswarmException(Exception):
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
#########################################
|
|
11
|
+
# API layer exceptions
|
|
12
|
+
#########################################
|
|
13
|
+
|
|
14
|
+
class PolyswarmAPIException(PolyswarmException):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TimeoutException(PolyswarmAPIException):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
#########################################
|
|
23
|
+
# Request layer exceptions
|
|
24
|
+
#########################################
|
|
25
|
+
|
|
26
|
+
class RequestException(PolyswarmException):
|
|
27
|
+
def __init__(self, request, *args):
|
|
28
|
+
super().__init__(*args)
|
|
29
|
+
self.request = request
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class UsageLimitsExceededException(RequestException):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class NotFoundException(RequestException):
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _normalise_sources(sources):
|
|
41
|
+
"""Coerce a server-supplied ``sources`` payload into a list of feed names.
|
|
42
|
+
|
|
43
|
+
``.sources`` is published as a list of strings and consumers iterate it, so a
|
|
44
|
+
bare ``'nsrl'`` cannot be handed through — it would iterate as characters.
|
|
45
|
+
The server sends a list of strings today; a feed-dict entry (the shape the
|
|
46
|
+
instance-level ``known_good`` field uses) yields ``feed['tool']`` defensively
|
|
47
|
+
rather than being dropped. Anything unrecognised becomes ``[]`` and is logged,
|
|
48
|
+
so a discard is never silent. The raw payload stays at ``exc.request.errors``.
|
|
49
|
+
"""
|
|
50
|
+
if isinstance(sources, str):
|
|
51
|
+
return [sources]
|
|
52
|
+
if isinstance(sources, list):
|
|
53
|
+
names = []
|
|
54
|
+
for source in sources:
|
|
55
|
+
if isinstance(source, str):
|
|
56
|
+
names.append(source)
|
|
57
|
+
elif isinstance(source, dict) and isinstance(source.get('tool'), str):
|
|
58
|
+
names.append(source['tool'])
|
|
59
|
+
else:
|
|
60
|
+
logger.warning('Dropping unrecognised known-good sources entry: %r', source)
|
|
61
|
+
return names
|
|
62
|
+
if sources is not None:
|
|
63
|
+
logger.warning('Dropping unrecognised known-good sources payload: %r', sources)
|
|
64
|
+
return []
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class KnownGoodWithheldException(NotFoundException):
|
|
68
|
+
"""404 for a known-good binary: the platform never stores or serves its bytes.
|
|
69
|
+
|
|
70
|
+
Subclasses ``NotFoundException`` so every existing ``except NotFoundException``
|
|
71
|
+
handler keeps working unchanged; catch this subclass only when the caller wants
|
|
72
|
+
to tell "withheld by design" apart from a plain "not found". The metadata the
|
|
73
|
+
platform holds about the artifact is still readable through the search /
|
|
74
|
+
instance endpoints.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
def __init__(self, request, *args, sources=None):
|
|
78
|
+
super().__init__(request, *args)
|
|
79
|
+
# Known-good feeds that flagged the hash (e.g. ``['nsrl']``); always a list
|
|
80
|
+
# of strings — normalised here, at the boundary, so the documented shape
|
|
81
|
+
# holds whatever the envelope carried. Empty when the server named none.
|
|
82
|
+
self.sources = _normalise_sources(sources)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class NoResultsException(RequestException):
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class FailedInstanceException(RequestException):
|
|
90
|
+
pass
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
#########################################
|
|
94
|
+
# Types layer exceptions
|
|
95
|
+
#########################################
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class TypeException(PolyswarmException):
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class MissingAPIInstanceException(TypeException):
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class InvalidJSONResponseException(TypeException):
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class DecodeErrorException(TypeException):
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class InvalidValueException(TypeException):
|
|
115
|
+
pass
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class ArtifactDeletedException(TypeException):
|
|
119
|
+
pass
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class InvalidYaraRulesException(TypeException):
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class NotImportedException(TypeException):
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
#########################################
|
|
131
|
+
# Warnings
|
|
132
|
+
#########################################
|
|
133
|
+
|
|
134
|
+
class APIWarning(Warning):
|
|
135
|
+
pass
|
|
@@ -275,9 +275,14 @@ class ArtifactInstance(core.BaseJsonResource, core.Hashable):
|
|
|
275
275
|
self.known_good_sources = sorted(
|
|
276
276
|
{feed['tool'] for feed in self.known_good if feed.get('tool')}
|
|
277
277
|
) if self.known_good else []
|
|
278
|
-
# Friendly
|
|
279
|
-
#
|
|
280
|
-
#
|
|
278
|
+
# Friendly REPORTED-state NAME (e.g. 'KNOWN_GOOD' / 'SETTLED' / 'STORED' /
|
|
279
|
+
# 'NOT_STORED'). The server derives it from its two-predicate known-good model
|
|
280
|
+
# (artifact-index specs/05): 'KNOWN_GOOD' while the file is currently known-good,
|
|
281
|
+
# 'NOT_STORED' for a submission it declined as known-good whose hash is no longer
|
|
282
|
+
# currently known-good (nothing was ever stored; a fresh submit works), otherwise
|
|
283
|
+
# the instance's own persisted state. Additive alongside the numeric
|
|
284
|
+
# bounty_state, and optional — older servers omit it, so .get() yields None
|
|
285
|
+
# (no behaviour change).
|
|
281
286
|
self.state = content.get('state')
|
|
282
287
|
|
|
283
288
|
# ArtifactInstance fields
|
|
@@ -151,6 +151,20 @@ class PolyswarmSession:
|
|
|
151
151
|
response.read()
|
|
152
152
|
_raise_for_status(response, request)
|
|
153
153
|
|
|
154
|
+
if response.status_code == 204:
|
|
155
|
+
# A 204 is the "no data / absent" signal on a download: the request
|
|
156
|
+
# succeeded but there is nothing to write ("the request worked, but
|
|
157
|
+
# there was no matching artifact, so nothing was returned"). The
|
|
158
|
+
# shared ``parse_response`` maps a parser-backed 204 to
|
|
159
|
+
# ``NoResultsException``; the streaming path bypasses ``parse_response``,
|
|
160
|
+
# so mirror that rule explicitly here. Without it, an absent artifact
|
|
161
|
+
# would be written out as a successful *empty* file instead of
|
|
162
|
+
# surfacing "no results".
|
|
163
|
+
raise exceptions.NoResultsException(
|
|
164
|
+
request,
|
|
165
|
+
"The request returned no results.",
|
|
166
|
+
)
|
|
167
|
+
|
|
154
168
|
pk = request.parser_kwargs or {}
|
|
155
169
|
handle, name, created = request.result_parser.open_destination(
|
|
156
170
|
pk.get("folder"),
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
class PolyswarmException(Exception):
|
|
2
|
-
pass
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
#########################################
|
|
6
|
-
# API layer exceptions
|
|
7
|
-
#########################################
|
|
8
|
-
|
|
9
|
-
class PolyswarmAPIException(PolyswarmException):
|
|
10
|
-
pass
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class TimeoutException(PolyswarmAPIException):
|
|
14
|
-
pass
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
#########################################
|
|
18
|
-
# Request layer exceptions
|
|
19
|
-
#########################################
|
|
20
|
-
|
|
21
|
-
class RequestException(PolyswarmException):
|
|
22
|
-
def __init__(self, request, *args):
|
|
23
|
-
super().__init__(*args)
|
|
24
|
-
self.request = request
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class UsageLimitsExceededException(RequestException):
|
|
28
|
-
pass
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class NotFoundException(RequestException):
|
|
32
|
-
pass
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class NoResultsException(RequestException):
|
|
36
|
-
pass
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
class FailedInstanceException(RequestException):
|
|
40
|
-
pass
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
#########################################
|
|
44
|
-
# Types layer exceptions
|
|
45
|
-
#########################################
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
class TypeException(PolyswarmException):
|
|
49
|
-
pass
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
class MissingAPIInstanceException(TypeException):
|
|
53
|
-
pass
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
class InvalidJSONResponseException(TypeException):
|
|
57
|
-
pass
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
class DecodeErrorException(TypeException):
|
|
61
|
-
pass
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class InvalidValueException(TypeException):
|
|
65
|
-
pass
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
class ArtifactDeletedException(TypeException):
|
|
69
|
-
pass
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
class InvalidYaraRulesException(TypeException):
|
|
73
|
-
pass
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
class NotImportedException(TypeException):
|
|
77
|
-
pass
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
#########################################
|
|
81
|
-
# Warnings
|
|
82
|
-
#########################################
|
|
83
|
-
|
|
84
|
-
class APIWarning(Warning):
|
|
85
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|